new: Add country ranking

bgpranking/abstractmanager.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 from abc import ABC, abstractmethod

bgpranking/libs/statsripe.py

@@ -38,22 +38,23 @@ class StatsRIPE():
         self.sourceapp = sourceapp
 
     def __time_to_text(self, query_time: TimeTypes) -> str:
-        if type(query_time, datetime):
+        if isinstance(query_time, datetime):
             return query_time.isoformat()
         return query_time
 
     def _get(self, method: str, parameters: dict) -> dict:
         parameters['sourceapp'] = self.sourceapp
         url = self.url.format(method=method, parameters='&'.join(['{}={}'.format(k, str(v).lower()) for k, v in parameters.items()]))
+        print(url)
         response = requests.get(url)
         return response.json()
 
-    async def network_info(self, ip: IPTypes) -> dict:
+    def network_info(self, ip: IPTypes) -> dict:
         parameters = {'resource': ip}
         return self._get('network-info', parameters)
 
-    async def prefix_overview(self, prefix: PrefixTypes, min_peers_seeing: int= 0,
+    def prefix_overview(self, prefix: PrefixTypes, min_peers_seeing: int= 0,
-                              max_related: int=0, query_time: TimeTypes=None) -> dict:
+                        max_related: int=0, query_time: TimeTypes=None) -> dict:
         parameters = {'resource': prefix}
         if min_peers_seeing:
             parameters['min_peers_seeing'] = min_peers_seeing
@@ -63,22 +64,19 @@ class StatsRIPE():
             parameters['query_time'] = self.__time_to_text(query_time)
         return self._get('prefix-overview', parameters)
 
-    async def ris_asns(self, query_time: TimeTypes=None, list_asns: bool=False, asn_types: ASNsTypes=ASNsTypes.undefined):
+    def ris_asns(self, query_time: TimeTypes=None, list_asns: bool=False, asn_types: ASNsTypes=ASNsTypes.undefined):
         parameters = {}
         if list_asns:
             parameters['list_asns'] = list_asns
         if asn_types:
             parameters['asn_types'] = asn_types.value
         if query_time:
-            if type(query_time, datetime):
+            parameters['query_time'] = self.__time_to_text(query_time)
-                parameters['query_time'] = query_time.isoformat()
-            else:
-                parameters['query_time'] = query_time
         return self._get('ris-asns', parameters)
 
-    async def ris_prefixes(self, asn: int, query_time: TimeTypes=None,
+    def ris_prefixes(self, asn: int, query_time: TimeTypes=None,
-                           list_prefixes: bool=False, types: ASNsTypes=ASNsTypes.undefined,
+                     list_prefixes: bool=False, types: ASNsTypes=ASNsTypes.undefined,
-                           af: AddressFamilies=AddressFamilies.undefined, noise: Noise=Noise.keep):
+                     af: AddressFamilies=AddressFamilies.undefined, noise: Noise=Noise.keep):
         parameters = {'resource': str(asn)}
         if query_time:
             parameters['query_time'] = self.__time_to_text(query_time)
@@ -91,3 +89,12 @@ class StatsRIPE():
         if noise:
             parameters['noise'] = noise.value
         return self._get('ris-prefixes', parameters)
+
+    def country_asns(self, country: str, details: int=0, query_time: TimeTypes=None):
+        parameters = {'resource': country}
+        if details:
+            parameters['lod'] = details
+        # FIXME: query_time makes the backend fail.
+        # if query_time:
+        #    parameters['query_time'] = self.__time_to_text(query_time)
+        return self._get('country-asns', parameters)

bgpranking/querying.py

@@ -12,6 +12,7 @@ from redis import StrictRedis
 
 from .libs.helpers import get_socket_path
 from .libs.exceptions import InvalidDateFormat
+from .libs.statsripe import StatsRIPE
 
 Dates = TypeVar('Dates', datetime.datetime, datetime.date, str)
 
@@ -62,10 +63,13 @@ class Querying():
         d = self.__normalize_date(date)
         if source:
             key = f'{d}|{source}|{asn}|{ipversion}'
-            return self.ranking.get(key)
+            r = self.ranking.get(key)
         else:
             key = f'{d}|asns|{ipversion}'
-            return self.ranking.zscore(key, asn)
+            r = self.ranking.zscore(key, asn)
+        if r:
+            return r
+        return 0
 
     def get_sources(self, date: Dates=datetime.date.today()):
         '''Get the sources availables for a specific day (default: today).'''
@@ -108,3 +112,31 @@ class Querying():
                 rank = 0
             to_return.insert(0, (d.isoformat(), rank))
         return to_return
+
+    def country_rank(self, country: str, date: Dates=datetime.date.today(), source: str='', ipversion: str='v4'):
+        ripe = StatsRIPE()
+        d = self.__normalize_date(date)
+        response = ripe.country_asns(country, query_time=d, details=1)
+        if (not response.get('data') or not response['data'].get('countries') or not
+                response['data']['countries'][0].get('routed')):
+            logging.warning(f'Invalid response: {response}')
+            # FIXME: return something
+            return
+        return sum([self.asn_rank(asn, d, source, ipversion) for asn in response['data']['countries'][0]['routed']])
+
+    def country_history(self, country: str, period: int=30, source: str='', ipversion: str='v4', date: Dates=datetime.date.today()):
+        to_return = []
+
+        if isinstance(date, str):
+            date = parse(date).date()
+        if date + timedelta(days=period / 3) > datetime.date.today():
+            # the period to display will be around the date passed at least 2/3 before the date, at most 1/3 after
+            date = datetime.date.today()
+
+        for i in range(period):
+            d = date - timedelta(days=i)
+            rank = self.country_rank(country, d, source, ipversion)
+            if rank is None:
+                rank = 0
+            to_return.insert(0, (d.isoformat(), rank))
+        return to_return

website/web/__init__.py

@@ -54,6 +54,7 @@ def index():
     q = Querying()
     sources = q.get_sources(date=session['date'])
     session.pop('asn', None)
+    session.pop('country', None)
     ranks = q.asns_global_ranking(limit=100, **session)
     descriptions = [q.get_asn_descriptions(int(asn)) for asn, rank in ranks]
     r = zip(ranks, descriptions)
@@ -63,6 +64,7 @@ def index():
 @app.route('/asn', methods=['GET', 'POST'])
 def asn_details():
     load_session()
+    session.pop('country', None)
     q = Querying()
     asn_descriptions = q.get_asn_descriptions(asn=session['asn'], all_descriptions=True)
     sources = q.get_sources(date=session['date'])
@@ -80,5 +82,22 @@ def asn_details():
 @app.route('/asn_history', methods=['GET', 'POST'])
 def asn_history():
     load_session()
+    session.pop('country', None)
     q = Querying()
     return json.dumps(q.get_asn_history(**session))
+
+
+@app.route('/country_history', methods=['GET', 'POST'])
+def country_history():
+    load_session()
+    q = Querying()
+    return json.dumps(q.country_history(**session))
+
+
+@app.route('/country', methods=['GET', 'POST'])
+def country():
+    load_session()
+    q = Querying()
+    sources = q.get_sources(date=session['date'])
+    session.pop('asn', None)
+    return render_template('country.html', sources=sources, **session)

website/web/static/linegraph.js

@@ -1,97 +1,99 @@
-var canvas = document.querySelector("canvas"),
+function linegraph(call_path) {
-    context = canvas.getContext("2d");
+    var canvas = document.querySelector("canvas"),
+        context = canvas.getContext("2d");
 
-// set the dimensions and margins of the graph
+    // set the dimensions and margins of the graph
-var margin = {top: 20, right: 20, bottom: 30, left: 50},
+    var margin = {top: 20, right: 20, bottom: 30, left: 50},
-    width = canvas.width - margin.left - margin.right,
+        width = canvas.width - margin.left - margin.right,
-    height = canvas.height - margin.top - margin.bottom;
+        height = canvas.height - margin.top - margin.bottom;
 
-// parse the date / time
+    // parse the date / time
-var parseTime = d3.timeParse("%Y-%m-%d");
+    var parseTime = d3.timeParse("%Y-%m-%d");
 
-// set the ranges
+    // set the ranges
-var x = d3.scaleTime().range([0, width]);
+    var x = d3.scaleTime().range([0, width]);
-var y = d3.scaleLinear().range([height, 0]);
+    var y = d3.scaleLinear().range([height, 0]);
 
-// define the line
+    // define the line
-var line = d3.line()
+    var line = d3.line()
-    .x(function(d) { return x(parseTime(d[0])); })
+        .x(function(d) { return x(parseTime(d[0])); })
-    .y(function(d) { return y(d[1]); })
+        .y(function(d) { return y(d[1]); })
-    .curve(d3.curveStep)
+        .curve(d3.curveStep)
-    .context(context);
+        .context(context);
 
-context.translate(margin.left, margin.top);
+    context.translate(margin.left, margin.top);
 
-// Get the data
+    // Get the data
-d3.json("/asn_history", {credentials: 'same-origin'}).then(function(data) {
+    d3.json(call_path, {credentials: 'same-origin'}).then(function(data) {
-  x.domain(d3.extent(data, function(d) { return parseTime(d[0]); }));
+      x.domain(d3.extent(data, function(d) { return parseTime(d[0]); }));
-  y.domain(d3.extent(data, function(d) { return d[1]; }));
+      y.domain(d3.extent(data, function(d) { return d[1]; }));
 
-  xAxis();
+      xAxis();
-  yAxis();
+      yAxis();
 
-  context.beginPath();
+      context.beginPath();
-  line(data);
+      line(data);
-  context.lineWidth = 1.5;
+      context.lineWidth = 1.5;
-  context.strokeStyle = "steelblue";
+      context.strokeStyle = "steelblue";
-  context.stroke();
+      context.stroke();
-});
+    });
 
-function xAxis() {
+    function xAxis() {
-  var tickCount = 10,
+      var tickCount = 10,
-      tickSize = .1,
+          tickSize = .1,
-      ticks = x.ticks(tickCount),
+          ticks = x.ticks(tickCount),
-      tickFormat = x.tickFormat();
+          tickFormat = x.tickFormat();
 
-  context.beginPath();
+      context.beginPath();
-  ticks.forEach(function(d) {
+      ticks.forEach(function(d) {
-    context.moveTo(x(d), height);
+        context.moveTo(x(d), height);
-    context.lineTo(x(d), height + tickSize);
+        context.lineTo(x(d), height + tickSize);
-  });
+      });
-  context.strokeStyle = "black";
+      context.strokeStyle = "black";
-  context.stroke();
+      context.stroke();
 
-  context.textAlign = "center";
+      context.textAlign = "center";
-  context.textBaseline = "top";
+      context.textBaseline = "top";
-  ticks.forEach(function(d) {
+      ticks.forEach(function(d) {
-    context.fillText(tickFormat(d), x(d), height + tickSize);
+        context.fillText(tickFormat(d), x(d), height + tickSize);
-  });
+      });
-}
+    }
-
+
-function yAxis() {
+    function yAxis() {
-  var tickCount = 20,
+      var tickCount = 20,
-      tickSize = 1,
+          tickSize = 1,
-      tickPadding = 1,
+          tickPadding = 1,
-      ticks = y.ticks(tickCount),
+          ticks = y.ticks(tickCount),
-      tickFormat = y.tickFormat(tickCount);
+          tickFormat = y.tickFormat(tickCount);
-
+
-  context.beginPath();
+      context.beginPath();
-  ticks.forEach(function(d) {
+      ticks.forEach(function(d) {
-    context.moveTo(0, y(d));
+        context.moveTo(0, y(d));
-    context.lineTo(-6, y(d));
+        context.lineTo(-6, y(d));
-  });
+      });
-  context.strokeStyle = "black";
+      context.strokeStyle = "black";
-  context.stroke();
+      context.stroke();
-
+
-  context.beginPath();
+      context.beginPath();
-  context.moveTo(-tickSize, 0);
+      context.moveTo(-tickSize, 0);
-  context.lineTo(0.5, 0);
+      context.lineTo(0.5, 0);
-  context.lineTo(0.5, height);
+      context.lineTo(0.5, height);
-  context.lineTo(-tickSize, height);
+      context.lineTo(-tickSize, height);
-  context.strokeStyle = "black";
+      context.strokeStyle = "black";
-  context.stroke();
+      context.stroke();
-
+
-  context.textAlign = "right";
+      context.textAlign = "right";
-  context.textBaseline = "middle";
+      context.textBaseline = "middle";
-  ticks.forEach(function(d) {
+      ticks.forEach(function(d) {
-    context.fillText(tickFormat(d), -tickSize - tickPadding, y(d));
+        context.fillText(tickFormat(d), -tickSize - tickPadding, y(d));
-  });
+      });
-
+
-  context.save();
+      context.save();
-  context.rotate(-Math.PI / 2);
+      context.rotate(-Math.PI / 2);
-  context.textAlign = "right";
+      context.textAlign = "right";
-  context.textBaseline = "top";
+      context.textBaseline = "top";
-  context.font = "bold 10px sans-serif";
+      context.font = "bold 10px sans-serif";
-  context.fillText("Rank", -10, 10);
+      context.fillText("Rank", -10, 10);
-  context.restore();
+      context.restore();
+    }
 }

website/web/templates/asn.html

@@ -10,6 +10,7 @@
 {% block scripts %}
     {{ super() }}
     <script src='{{ url_for('static', filename='linegraph.js') }}'></script>
+    <script>linegraph('/asn_history');</script>
 {% endblock %}
 
 {% block content %}

website/web/templates/country.html

@@ -0,0 +1,22 @@
+{% extends "main.html" %}
+
+{% block head %}
+    {{ super() }}
+{% endblock %}
+
+
+{% block title %}Ranking - {{ country }}{% endblock %}
+
+{% block scripts %}
+    {{ super() }}
+    <script src='{{ url_for('static', filename='linegraph.js') }}'></script>
+    <script>linegraph('/country_history');</script>
+{% endblock %}
+
+{% block content %}
+  <center>
+      <h1>Ranking - {{country}}</h1></br></br>
+  </center>
+  {% include ['top_forms.html'] %}
+  <canvas width="1024" height="800"></canvas>
+{% endblock %}