new: Major rewrite of the website

bgpranking/querying.py

@@ -3,7 +3,6 @@
 
 from typing import TypeVar
 import datetime
-from enum import Enum
 from dateutil.parser import parse
 
 import logging
@@ -15,11 +14,6 @@ from .libs.exceptions import InvalidDateFormat
 Dates = TypeVar('Dates', datetime.datetime, datetime.date, str)
 
 
-class IPVersion(Enum):
-    v4 = 'v4'
-    v6 = 'v6'
-
-
 class Querying():
 
     def __init__(self, loglevel: int=logging.DEBUG):
@@ -43,34 +37,35 @@ class Querying():
             except ValueError:
                 raise InvalidDateFormat('Unable to parse the date. Should be YYYY-MM-DD.')
 
-    def asns_global_ranking(self, date: Dates=datetime.date.today(),
+    def asns_global_ranking(self, date: Dates=datetime.date.today(), source: str='', ipversion: str='v4', limit: int=100):
-                            ipversion: IPVersion=IPVersion.v4, limit: int=100):
         '''Aggregated ranking of all the ASNs known in the system, weighted by source.'''
         d = self.__normalize_date(date)
-        key = f'{d}|asns|{ipversion.value}'
+        if source:
+            key = f'{d}|{source}|asns|{ipversion}'
+        else:
+            key = f'{d}|asns|{ipversion}'
         return self.ranking.zrevrange(key, start=0, end=limit, withscores=True)
 
-    def asn_details(self, asn: int, date: Dates= datetime.date.today(), ipversion: IPVersion=IPVersion.v4):
+    def asn_details(self, asn: int, date: Dates= datetime.date.today(), source: str='', ipversion: str='v4'):
         '''Aggregated ranking of all the prefixes anounced by the given ASN, weighted by source.'''
         d = self.__normalize_date(date)
-        key = f'{d}|{asn}|{ipversion.value}'
+        if source:
+            key = f'{d}|{source}|{asn}|rank{ipversion}|prefixes'
+        else:
+            key = f'{d}|{asn}|{ipversion}'
         return self.ranking.zrevrange(key, start=0, end=-1, withscores=True)
 
-    def asn_rank(self, asn: int, date: Dates= datetime.date.today(), ipversion: IPVersion=IPVersion.v4):
+    def asn_rank(self, asn: int, date: Dates= datetime.date.today(), source: str='', ipversion: str='v4'):
         '''Get the rank of a single ASN, weighted by source.'''
         d = self.__normalize_date(date)
-        key = f'{d}|asns|{ipversion.value}'
+        if source:
+            key = f'{d}|{source}|{asn}|rank{ipversion}'
+        else:
+            key = f'{d}|asns|{ipversion}'
         return self.ranking.zscore(key, asn)
 
-    def asn_rank_by_source(self, asn: int, source: str, date: Dates= datetime.date.today(), ipversion: IPVersion=IPVersion.v4):
+    def get_sources(self, date: Dates= datetime.date.today()):
-        '''Get the rank of a single ASN, not weighted by source.'''
+        '''Get the sources availables for a specific day (default: today).'''
         d = self.__normalize_date(date)
-        key = f'{d}|{source}|{asn}|rank{ipversion.value}'
+        key = f'{d}|sources'
-        return self.ranking.get(key)
+        return self.storage.smembers(key)
-
-    def asn_details_by_source(self, source: str, asn: int, date: Dates= datetime.date.today(),
-                              ipversion: IPVersion=IPVersion.v4):
-        '''Get the rank of all the prefixes announced by an ASN, not weighted by source.'''
-        d = self.__normalize_date(date)
-        key = f'{d}|{source}|{asn}|rank{ipversion.value}|prefixes'
-        return self.ranking.zrevrange(key, start=0, end=-1, withscores=True)

bgpranking/ranking.py

@@ -37,6 +37,9 @@ class Ranking():
         r_pipeline = self.ranking.pipeline()
         for source in self.storage.smembers(f'{today}|sources'):
             self.logger.info(f'{today} - Ranking source: {source}')
+            source_aggregation_key_v4 = f'{today}|{source}|asns|v4'
+            source_aggregation_key_v6 = f'{today}|{source}|asns|v6'
+            to_delete.update([source_aggregation_key_v4, source_aggregation_key_v6])
             for asn in self.storage.smembers(f'{today}|{source}'):
                 prefixes_aggregation_key_v4 = f'{today}|{asn}|v4'
                 prefixes_aggregation_key_v6 = f'{today}|{asn}|v6'
@@ -65,11 +68,13 @@ class Ranking():
                     if asn_rank_v4:
                         r_pipeline.set(f'{today}|{source}|{asn}|rankv4', asn_rank_v4)
                         r_pipeline.zincrby(asns_aggregation_key_v4, asn, asn_rank_v4)
+                        r_pipeline.zadd(source_aggregation_key_v4, asn_rank_v4, asn)
                 if v6count:
                     asn_rank_v6 /= float(v6count)
                     if asn_rank_v6:
                         r_pipeline.set(f'{today}|{source}|{asn}|rankv6', asn_rank_v6)
                         r_pipeline.zincrby(asns_aggregation_key_v6, asn, asn_rank_v6)
+                        r_pipeline.zadd(source_aggregation_key_v6, asn_rank_v4, asn)
         self.ranking.delete(*to_delete)
         r_pipeline.execute()
 

website/3drparty.sh

@@ -3,5 +3,7 @@
 set -e
 set -x
 
+mkdir -p web/static/
+
 wget https://code.jquery.com/ui/1.12.1/jquery-ui.js -O web/static/jquery-ui.js
 wget https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css -O web/static/jquery-ui.css

website/requirements.txt

@@ -1,4 +1,3 @@
 # Web thing
 flask
 flask-bootstrap
-Flask-Datepicker

website/web/__init__.py

@@ -1,34 +1,60 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from flask import Flask, render_template, request
+from flask import Flask, render_template, request, session
 from flask_bootstrap import Bootstrap
-from flask_datepicker import datepicker
 
 from bgpranking.querying import Querying
+from pathlib import Path
+from datetime import date, timedelta
+
 
 app = Flask(__name__)
 
+app.secret_key = '\xeb\xfd\x1b\xee\xed<\xa5~\xd5H\x85\x00\xa5r\xae\x80t5@\xa2&>\x03S'
+
 Bootstrap(app)
 app.config['BOOTSTRAP_SERVE_LOCAL'] = True
-datepicker(app=app, local=['static/jquery-ui.js', 'static/jquery-ui.css'])
 
-app.debug = True
+jquery_js = Path('static', 'jquery-ui.js')
+jquery_css = Path('static', 'jquery-ui.css')
 
 
-@app.route('/', methods=['GET'])
+def load_session():
+    if request.method == 'POST':
+        d = request.form
+    elif request.method == 'GET':
+        d = request.args
+
+    if 'date' in d:
+        session['date'] = d['date']
+    if 'ipversion' in d:
+        session['ipversion'] = d['ipversion']
+    if 'source' in d:
+        session['source'] = d['source']
+    if 'asn' in d:
+        session['asn'] = d['asn']
+    set_default_date_session()
+
+
+def set_default_date_session():
+    if 'date' not in session:
+        session['date'] = (date.today() - timedelta(days=1)).isoformat()
+
+
+@app.route('/', methods=['GET', 'POST'])
 def index():
+    load_session()
     q = Querying()
-    ranks = q.asns_global_ranking(limit=-1)
+    sources = q.get_sources(date=session['date'])
-    return render_template('index.html', ranks=ranks)
+    session.pop('asn', None)
+    ranks = q.asns_global_ranking(limit=-1, **session)
+    return render_template('index.html', ranks=ranks, sources=sources, **session)
 
 
 @app.route('/asn', methods=['GET', 'POST'])
 def asn_details():
+    load_session()
     q = Querying()
-    if request.method == 'POST':
+    ranks = q.asn_details(**session)
-        asn = request.form['asn']
+    return render_template('asn.html', ranks=ranks, **session)
-    if request.method == 'GET':
-        asn = request.args['asn']
-    ranks = q.asn_details(asn)
-    return render_template('asn.html', asn=asn, ranks=ranks)

website/web/templates/index.html

@@ -6,13 +6,35 @@
   <center>
     <h1>BGP Ranking</h1></br></br>
   </center>
+  <p>
+      <form style="width:300px; display:inline-block;" action="" method=post>
+          <input name="date" type="date" value="{{ date }}">
+          <input type="submit" value="Set date">
+      </form>
+      <form style="width:300px; display:inline-block;" action="" method=post>
+          <select name="ipversion">
+              <option value="v4" {% if ipversion == 'v4' %} selected="selected"{% endif %}>v4</option>
+              <option value="v6" {% if ipversion == 'v6' %} selected="selected"{% endif %}>v6</option>
+          </select>
+          <input type="submit" value="Set IP version">
+      </form>
+      <form style="width:300px; display:inline-block;" action="" method=post>
+        <select name="source">
+          <option value="" {% if not source %} selected="selected"{% endif %}>all</option>
+          {% for s in sources %}
+          <option value="{{ s }}" {% if source == s %} selected="selected"{% endif %}>{{ s }}</option>
+          {% endfor %}
+        </select>
+        <input type="submit" value="Set source">
+      </form>
+  <p>
+
+  <br/>
   <form action="{{ url_for('asn_details') }}" method=post>
       ASN to search: <input type=number name=asn>
       <input type="submit" value="Search">
   </form>
-  <form class="verticalform">
+  <br/>
-      <input type="text" class="form-control dp" />
-  </form>
   <table class="table">
       <tr>
           <th>ASN</th>

website/web/templates/main.html

@@ -2,8 +2,6 @@
 
 {% block scripts %}
     {{ super() }}
-    {{ datepicker.loader() }}
-    {{ datepicker.picker(id=".dp") }}
 {% endblock %}
 
 {% block head %}