D4
/
BGP-Ranking
mirror of https://github.com/D4-project/BGP-Ranking
2
0
Fork 0
Code Issues Releases Wiki Activity

new: Allow to select multiple sources, caching.

pull/12/head
Raphaël Vinot 2018-07-31 14:21:06 +02:00
parent 9cab8d1196
commit 0398d2f2b9
5 changed files with 84 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
bgpranking/querying.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from typing import TypeVar
from typing import TypeVar, Union
import datetime
from datetime import timedelta
from dateutil.parser import parse
@ -24,6 +24,7 @@ class Querying():
self.storage = StrictRedis(unix_socket_path=get_socket_path('storage'), decode_responses=True)
self.ranking = StrictRedis(unix_socket_path=get_socket_path('storage'), db=1, decode_responses=True)
self.asn_meta = StrictRedis(unix_socket_path=get_socket_path('storage'), db=2, decode_responses=True)
self.cache = StrictRedis(unix_socket_path=get_socket_path('ris'), db=1, decode_responses=True)
def __init_logger(self, loglevel: int):
self.logger = logging.getLogger(f'{self.__class__.__name__}')
@ -40,33 +41,77 @@ class Querying():
except ValueError:
raise InvalidDateFormat('Unable to parse the date. Should be YYYY-MM-DD.')
def asns_global_ranking(self, date: Dates=datetime.date.today(), source: str='', ipversion: str='v4', limit: int=100):
def ranking_cache_wrapper(self, key):
if not self.cache.exists(key):
if self.ranking.exists(key):
key_dump = self.ranking.dump(key)
# Cache for 10 hours
self.cache.restore(key, 36000, key_dump, True)
def asns_global_ranking(self, date: Dates=datetime.date.today(), source: Union[list, str]='',
ipversion: str='v4', limit: int=100):
'''Aggregated ranking of all the ASNs known in the system, weighted by source.'''
d = self.__normalize_date(date)
if source:
key = f'{d}|{source}|asns|{ipversion}'
if isinstance(source, list):
keys = []
for s in source:
key = f'{d}|{s}|asns|{ipversion}'
self.ranking_cache_wrapper(key)
keys.append(key)
# union the ranked sets
key = '|'.join(sorted(source)) + f'|{d}|asns|{ipversion}'
if not self.cache.exists(key):
self.cache.zunionstore(key, keys)
else:
key = f'{d}|{source}|asns|{ipversion}'
else:
key = f'{d}|asns|{ipversion}'
return self.ranking.zrevrange(key, start=0, end=limit, withscores=True)
self.ranking_cache_wrapper(key)
return self.cache.zrevrange(key, start=0, end=limit, withscores=True)
def asn_details(self, asn: int, date: Dates= datetime.date.today(), source: str='', ipversion: str='v4'):
def asn_details(self, asn: int, date: Dates= datetime.date.today(), source: Union[list, str]='',
ipversion: str='v4'):
'''Aggregated ranking of all the prefixes anounced by the given ASN, weighted by source.'''
d = self.__normalize_date(date)
if source:
key = f'{d}|{source}|{asn}|{ipversion}|prefixes'
if isinstance(source, list):
keys = []
for s in source:
key = f'{d}|{s}|{asn}|{ipversion}|prefixes'
self.ranking_cache_wrapper(key)
keys.append(key)
# union the ranked sets
key = '|'.join(sorted(source)) + f'|{d}|{asn}|{ipversion}'
if not self.cache.exists(key):
self.cache.zunionstore(key, keys)
else:
key = f'{d}|{source}|{asn}|{ipversion}|prefixes'
else:
key = f'{d}|{asn}|{ipversion}'
return self.ranking.zrevrange(key, start=0, end=-1, withscores=True)
self.ranking_cache_wrapper(key)
return self.cache.zrevrange(key, start=0, end=-1, withscores=True)
def asn_rank(self, asn: int, date: Dates=datetime.date.today(), source: str='', ipversion: str='v4'):
def asn_rank(self, asn: int, date: Dates=datetime.date.today(), source: Union[list, str]='',
ipversion: str='v4'):
'''Get the rank of a single ASN, weighted by source.'''
d = self.__normalize_date(date)
if source:
key = f'{d}|{source}|{asn}|{ipversion}'
r = self.ranking.get(key)
if isinstance(source, list):
keys = []
for s in source:
key = f'{d}|{s}|{asn}|{ipversion}'
self.ranking_cache_wrapper(key)
keys.append(key)
r = sum(float(self.cache.get(key)) for key in keys if self.cache.exists(key))
else:
key = f'{d}|{source}|{asn}|{ipversion}'
self.ranking_cache_wrapper(key)
r = self.cache.get(key)
else:
key = f'{d}|asns|{ipversion}'
r = self.ranking.zscore(key, asn)
self.ranking_cache_wrapper(key)
r = self.cache.zscore(key, asn)
if r:
return float(r)
return 0
@ -83,9 +128,13 @@ class Querying():
return descriptions
return descriptions[sorted(descriptions.keys(), reverse=True)[0]]
def get_prefix_ips(self, asn: int, prefix: str, date: Dates=datetime.date.today(), source: str='', ipversion: str='v4'):
def get_prefix_ips(self, asn: int, prefix: str, date: Dates=datetime.date.today(),
source: Union[list, str]='', ipversion: str='v4'):
if source:
sources = [source]
if isinstance(source, list):
sources = source
else:
sources = [source]
else:
sources = self.get_sources(date)
prefix_ips = defaultdict(list)
@ -96,7 +145,8 @@ class Querying():
[prefix_ips[ip].append(source) for ip in ips]
return prefix_ips
def get_asn_history(self, asn: int, period: int=100, source: str='', ipversion: str='v4', date: Dates=datetime.date.today()):
def get_asn_history(self, asn: int, period: int=100, source: Union[list, str]='',
ipversion: str='v4', date: Dates=datetime.date.today()):
to_return = []
if isinstance(date, str):
@ -113,7 +163,8 @@ class Querying():
to_return.insert(0, (d.isoformat(), rank))
return to_return
def country_rank(self, country: str, date: Dates=datetime.date.today(), source: str='', ipversion: str='v4'):
def country_rank(self, country: str, date: Dates=datetime.date.today(), source: Union[list, str]='',
ipversion: str='v4'):
ripe = StatsRIPE()
d = self.__normalize_date(date)
response = ripe.country_asns(country, query_time=d, details=1)
@ -128,7 +179,8 @@ class Querying():
daily_sum = sum(ranks)
return daily_sum, to_return
def country_history(self, country: str, period: int=30, source: str='', ipversion: str='v4', date: Dates=datetime.date.today()):
def country_history(self, country: str, period: int=30, source: Union[list, str]='',
ipversion: str='v4', date: Dates=datetime.date.today()):
to_return = []
if isinstance(date, str):

5
website/3drparty.sh
View File

@ -6,3 +6,8 @@ set -x
mkdir -p web/static/
wget https://d3js.org/d3.v5.js -O web/static/d3.v5.js
BOOTSTRAP_SELECT="1.12.4"
wget https://cdnjs.cloudflare.com/ajax/libs/bootstrap-select/${BOOTSTRAP_SELECT}/css/bootstrap-select.min.css -O web/static/bootstrap-select.min.css
wget https://cdnjs.cloudflare.com/ajax/libs/bootstrap-select/${BOOTSTRAP_SELECT}/js/bootstrap-select.min.js -O web/static/bootstrap-select.min.js

7
website/web/__init__.py
View File

@ -39,12 +39,15 @@ def load_session():
if 'ipversion' in d:
session['ipversion'] = d['ipversion']
if 'source' in d:
session['source'] = d['source']
if '_all' in d.getlist('source'):
session.pop('source', None)
else:
session['source'] = d.getlist('source')
if 'asn' in d:
session['asn'] = d['asn']
session.pop('country', None)
elif 'country' in d:
session['country'] = d['country']
session['country'] = d.getlist('country')
session.pop('asn', None)
set_default_date_session()

2
website/web/templates/main.html
View File

@ -3,9 +3,11 @@
{% block scripts %}
{{ super() }}
<script src='{{ url_for('static', filename='d3.v5.js') }}'></script>
<script src='{{ url_for('static', filename='bootstrap-select.min.js') }}'></script>
{% endblock %}
{% block head %}
<link rel="stylesheet" href="{{ url_for('static', filename='bootstrap-select.min.css') }}">
{{ super() }}
{% endblock %}

8
website/web/templates/top_forms.html
View File

@ -11,10 +11,10 @@
<button type="submit" class="btn btn-primary my-1">IP version</button>
</form>
<form class="form-inline" style="width:400px; display:inline-block;" action="" method=post>
<select name="source" class="form-control">
<option value="" {% if not source %} selected="selected"{% endif %}>all</option>
<select name="source" class="selectpicker" multiple>
<option value="_all" {% if not source %} selected="selected"{% endif %}>all</option>
{% for s in sources %}
<option value="{{ s }}" {% if source == s %} selected="selected"{% endif %}>{{ s }}</option>
<option value="{{ s }}" {% if s in source %} selected="selected"{% endif %}>{{ s }}</option>
{% endfor %}
</select>
<input type="submit" class="btn btn-primary my-1"value="Set source">
@ -24,7 +24,7 @@
<button type="submit" class="btn btn-primary my-1">Search ASN</button>
</form>
<form class="form-inline"style="width:500px; display:inline-block;" action="{{ url_for('country') }}" method=post>
<select name="country" class="form-control">
<select name="country" class="selectpicker" multiple>
<option value="" {% if not country %} selected="selected"{% endif %}>all</option>
{% for cc, name in countries %}
<option value="{{ cc }}" {% if country == cc %} selected="selected"{% endif %}>{{ name }}</option>