new: Add country ranking

pull/12/head
Raphaël Vinot 2018-07-27 14:33:25 +02:00
parent 85a6f6e223
commit 7883bdd3a9
7 changed files with 183 additions and 100 deletions

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from abc import ABC, abstractmethod

View File

@ -38,21 +38,22 @@ class StatsRIPE():
self.sourceapp = sourceapp
def __time_to_text(self, query_time: TimeTypes) -> str:
if type(query_time, datetime):
if isinstance(query_time, datetime):
return query_time.isoformat()
return query_time
def _get(self, method: str, parameters: dict) -> dict:
parameters['sourceapp'] = self.sourceapp
url = self.url.format(method=method, parameters='&'.join(['{}={}'.format(k, str(v).lower()) for k, v in parameters.items()]))
print(url)
response = requests.get(url)
return response.json()
async def network_info(self, ip: IPTypes) -> dict:
def network_info(self, ip: IPTypes) -> dict:
parameters = {'resource': ip}
return self._get('network-info', parameters)
async def prefix_overview(self, prefix: PrefixTypes, min_peers_seeing: int= 0,
def prefix_overview(self, prefix: PrefixTypes, min_peers_seeing: int= 0,
max_related: int=0, query_time: TimeTypes=None) -> dict:
parameters = {'resource': prefix}
if min_peers_seeing:
@ -63,20 +64,17 @@ class StatsRIPE():
parameters['query_time'] = self.__time_to_text(query_time)
return self._get('prefix-overview', parameters)
async def ris_asns(self, query_time: TimeTypes=None, list_asns: bool=False, asn_types: ASNsTypes=ASNsTypes.undefined):
def ris_asns(self, query_time: TimeTypes=None, list_asns: bool=False, asn_types: ASNsTypes=ASNsTypes.undefined):
parameters = {}
if list_asns:
parameters['list_asns'] = list_asns
if asn_types:
parameters['asn_types'] = asn_types.value
if query_time:
if type(query_time, datetime):
parameters['query_time'] = query_time.isoformat()
else:
parameters['query_time'] = query_time
parameters['query_time'] = self.__time_to_text(query_time)
return self._get('ris-asns', parameters)
async def ris_prefixes(self, asn: int, query_time: TimeTypes=None,
def ris_prefixes(self, asn: int, query_time: TimeTypes=None,
list_prefixes: bool=False, types: ASNsTypes=ASNsTypes.undefined,
af: AddressFamilies=AddressFamilies.undefined, noise: Noise=Noise.keep):
parameters = {'resource': str(asn)}
@ -91,3 +89,12 @@ class StatsRIPE():
if noise:
parameters['noise'] = noise.value
return self._get('ris-prefixes', parameters)
def country_asns(self, country: str, details: int=0, query_time: TimeTypes=None):
parameters = {'resource': country}
if details:
parameters['lod'] = details
# FIXME: query_time makes the backend fail.
# if query_time:
# parameters['query_time'] = self.__time_to_text(query_time)
return self._get('country-asns', parameters)

View File

@ -12,6 +12,7 @@ from redis import StrictRedis
from .libs.helpers import get_socket_path
from .libs.exceptions import InvalidDateFormat
from .libs.statsripe import StatsRIPE
Dates = TypeVar('Dates', datetime.datetime, datetime.date, str)
@ -62,10 +63,13 @@ class Querying():
d = self.__normalize_date(date)
if source:
key = f'{d}|{source}|{asn}|{ipversion}'
return self.ranking.get(key)
r = self.ranking.get(key)
else:
key = f'{d}|asns|{ipversion}'
return self.ranking.zscore(key, asn)
r = self.ranking.zscore(key, asn)
if r:
return r
return 0
def get_sources(self, date: Dates=datetime.date.today()):
'''Get the sources availables for a specific day (default: today).'''
@ -108,3 +112,31 @@ class Querying():
rank = 0
to_return.insert(0, (d.isoformat(), rank))
return to_return
def country_rank(self, country: str, date: Dates=datetime.date.today(), source: str='', ipversion: str='v4'):
ripe = StatsRIPE()
d = self.__normalize_date(date)
response = ripe.country_asns(country, query_time=d, details=1)
if (not response.get('data') or not response['data'].get('countries') or not
response['data']['countries'][0].get('routed')):
logging.warning(f'Invalid response: {response}')
# FIXME: return something
return
return sum([self.asn_rank(asn, d, source, ipversion) for asn in response['data']['countries'][0]['routed']])
def country_history(self, country: str, period: int=30, source: str='', ipversion: str='v4', date: Dates=datetime.date.today()):
to_return = []
if isinstance(date, str):
date = parse(date).date()
if date + timedelta(days=period / 3) > datetime.date.today():
# the period to display will be around the date passed at least 2/3 before the date, at most 1/3 after
date = datetime.date.today()
for i in range(period):
d = date - timedelta(days=i)
rank = self.country_rank(country, d, source, ipversion)
if rank is None:
rank = 0
to_return.insert(0, (d.isoformat(), rank))
return to_return

View File

@ -54,6 +54,7 @@ def index():
q = Querying()
sources = q.get_sources(date=session['date'])
session.pop('asn', None)
session.pop('country', None)
ranks = q.asns_global_ranking(limit=100, **session)
descriptions = [q.get_asn_descriptions(int(asn)) for asn, rank in ranks]
r = zip(ranks, descriptions)
@ -63,6 +64,7 @@ def index():
@app.route('/asn', methods=['GET', 'POST'])
def asn_details():
load_session()
session.pop('country', None)
q = Querying()
asn_descriptions = q.get_asn_descriptions(asn=session['asn'], all_descriptions=True)
sources = q.get_sources(date=session['date'])
@ -80,5 +82,22 @@ def asn_details():
@app.route('/asn_history', methods=['GET', 'POST'])
def asn_history():
load_session()
session.pop('country', None)
q = Querying()
return json.dumps(q.get_asn_history(**session))
@app.route('/country_history', methods=['GET', 'POST'])
def country_history():
load_session()
q = Querying()
return json.dumps(q.country_history(**session))
@app.route('/country', methods=['GET', 'POST'])
def country():
load_session()
q = Querying()
sources = q.get_sources(date=session['date'])
session.pop('asn', None)
return render_template('country.html', sources=sources, **session)

View File

@ -1,3 +1,4 @@
function linegraph(call_path) {
var canvas = document.querySelector("canvas"),
context = canvas.getContext("2d");
@ -23,7 +24,7 @@ var line = d3.line()
context.translate(margin.left, margin.top);
// Get the data
d3.json("/asn_history", {credentials: 'same-origin'}).then(function(data) {
d3.json(call_path, {credentials: 'same-origin'}).then(function(data) {
x.domain(d3.extent(data, function(d) { return parseTime(d[0]); }));
y.domain(d3.extent(data, function(d) { return d[1]; }));
@ -95,3 +96,4 @@ function yAxis() {
context.fillText("Rank", -10, 10);
context.restore();
}
}

View File

@ -10,6 +10,7 @@
{% block scripts %}
{{ super() }}
<script src='{{ url_for('static', filename='linegraph.js') }}'></script>
<script>linegraph('/asn_history');</script>
{% endblock %}
{% block content %}

View File

@ -0,0 +1,22 @@
{% extends "main.html" %}
{% block head %}
{{ super() }}
{% endblock %}
{% block title %}Ranking - {{ country }}{% endblock %}
{% block scripts %}
{{ super() }}
<script src='{{ url_for('static', filename='linegraph.js') }}'></script>
<script>linegraph('/country_history');</script>
{% endblock %}
{% block content %}
<center>
<h1>Ranking - {{country}}</h1></br></br>
</center>
{% include ['top_forms.html'] %}
<canvas width="1024" height="800"></canvas>
{% endblock %}