chg: Sync code with prod

bgpranking/asn_descriptions.py

@@ -21,7 +21,7 @@ class ASNDescriptions():
         safe_create_dir(self.directory)
         self.archives = self.directory / 'archive'
         safe_create_dir(self.archives)
-        self.url = 'http://www.cidr-report.org/as2.0/autnums.html'
+        self.url = 'https://www.cidr-report.org/as2.0/autnums.html'
 
     def __init_logger(self, loglevel):
         self.logger = logging.getLogger(f'{self.__class__.__name__}')

bgpranking/config/bgpranking.json

@@ -1,3 +1,3 @@
 {
-  "ipasnhistory_url": "http://127.0.0.1:5176/"
+  "ipasnhistory_url": "https://ipasnhistory.circl.lu/"
 }

bgpranking/dbinsert.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 import logging
+import time
 from redis import StrictRedis
 from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path, get_ipasn, sanity_check_ipasn
 
@@ -29,8 +30,15 @@ class DatabaseInsert():
 
         set_running(self.__class__.__name__)
         while True:
-            if shutdown_requested() or not self.ipasn.is_up:
+            if shutdown_requested():
                 break
+            try:
+                if not self.ipasn.is_up:
+                    break
+            except Exception:
+                self.logger.warning('Unable to query ipasnhistory')
+                time.sleep(10)
+                continue
             uuids = self.redis_sanitized.spop('to_insert', 100)
             if not uuids:
                 break
@@ -52,7 +60,8 @@ class DatabaseInsert():
                 self.logger.exception('Mass query in IPASN History failed, trying again later.')
                 # Rollback the spop
                 self.redis_sanitized.sadd('to_insert', *uuids)
-                break
+                time.sleep(10)
+                continue
             retry = []
             done = []
             ardb_pipeline = self.ardb_storage.pipeline(transaction=False)

bgpranking/sanitizer.py

@@ -51,6 +51,10 @@ class Sanitizer():
                 except ValueError:
                     self.logger.info(f"Invalid IP address: {data['ip']}")
                     continue
+                except KeyError:
+                    self.logger.info(f"Invalid entry {data}")
+                    continue
+
                 if not ip.is_global:
                     self.logger.info(f"The IP address {data['ip']} is not global")
                     continue

website/web/__init__.py

@@ -188,8 +188,13 @@ def ipasn_history_proxy(path):
     config, general_config_file = load_general_config()
     if 'ipasnhistory_url' not in config:
         raise MissingConfigEntry(f'"ipasnhistory_url" is missing in {general_config_file}.')
-    proxied_url = urljoin(config['ipasnhistory_url'],
-                          request.full_path.replace('/ipasn_history', ''))
+
+    path_for_ipasnhistory = request.full_path.replace('/ipasn_history', '')
+    if '/?' in path_for_ipasnhistory:
+        path_for_ipasnhistory = path_for_ipasnhistory.replace('/?', '/ip?')
+    print(path_for_ipasnhistory)
+    proxied_url = urljoin(config['ipasnhistory_url'], path_for_ipasnhistory)
+    print(proxied_url)
     if request.method in ['GET', 'HEAD']:
         to_return = requests.get(proxied_url).json()
     elif request.method == 'POST':