diff --git a/bgpranking/querying.py b/bgpranking/querying.py index 30243d4..da51623 100644 --- a/bgpranking/querying.py +++ b/bgpranking/querying.py @@ -42,7 +42,7 @@ class Querying(): except ValueError: raise InvalidDateFormat('Unable to parse the date. Should be YYYY-MM-DD.') - def ranking_cache_wrapper(self, key): + def _ranking_cache_wrapper(self, key): if not self.cache.exists(key): if self.ranking.exists(key): key_dump = self.ranking.dump(key) @@ -52,13 +52,17 @@ class Querying(): def asns_global_ranking(self, date: Dates=datetime.date.today(), source: Union[list, str]='', ipversion: str='v4', limit: int=100): '''Aggregated ranking of all the ASNs known in the system, weighted by source.''' + to_return = {'meta': {'ipversion': ipversion, 'limit': limit}, 'source': source, + 'response': set()} + d = self.__normalize_date(date) + to_return['meta']['date'] = d if source: if isinstance(source, list): keys = [] for s in source: key = f'{d}|{s}|asns|{ipversion}' - self.ranking_cache_wrapper(key) + self._ranking_cache_wrapper(key) keys.append(key) # union the ranked sets key = '|'.join(sorted(source)) + f'|{d}|asns|{ipversion}' @@ -68,19 +72,24 @@ class Querying(): key = f'{d}|{source}|asns|{ipversion}' else: key = f'{d}|asns|{ipversion}' - self.ranking_cache_wrapper(key) - return self.cache.zrevrange(key, start=0, end=limit, withscores=True) + self._ranking_cache_wrapper(key) + to_return['response'] = self.cache.zrevrange(key, start=0, end=limit, withscores=True) + return to_return def asn_details(self, asn: int, date: Dates= datetime.date.today(), source: Union[list, str]='', ipversion: str='v4'): '''Aggregated ranking of all the prefixes anounced by the given ASN, weighted by source.''' + to_return = {'meta': {'asn': asn, 'ipversion': ipversion, 'source': source}, + 'response': set()} + d = self.__normalize_date(date) + to_return['meta']['date'] = d if source: if isinstance(source, list): keys = [] for s in source: key = f'{d}|{s}|{asn}|{ipversion}|prefixes' - self.ranking_cache_wrapper(key) + self._ranking_cache_wrapper(key) keys.append(key) # union the ranked sets key = '|'.join(sorted(source)) + f'|{d}|{asn}|{ipversion}' @@ -90,84 +99,122 @@ class Querying(): key = f'{d}|{source}|{asn}|{ipversion}|prefixes' else: key = f'{d}|{asn}|{ipversion}' - self.ranking_cache_wrapper(key) - return self.cache.zrevrange(key, start=0, end=-1, withscores=True) + self._ranking_cache_wrapper(key) + to_return['response'] = self.cache.zrevrange(key, start=0, end=-1, withscores=True) + return to_return def asn_rank(self, asn: int, date: Dates=datetime.date.today(), source: Union[list, str]='', - ipversion: str='v4'): + ipversion: str='v4', with_position: bool=False): '''Get the rank of a single ASN, weighted by source.''' + to_return = {'meta': {'asn': asn, 'ipversion': ipversion, + 'source': source, 'with_position': with_position}, + 'response': 0.0} + d = self.__normalize_date(date) + to_return['meta']['date'] = d if source: + to_return['meta']['source'] = source if isinstance(source, list): keys = [] for s in source: key = f'{d}|{s}|{asn}|{ipversion}' - self.ranking_cache_wrapper(key) + self._ranking_cache_wrapper(key) keys.append(key) r = sum(float(self.cache.get(key)) for key in keys if self.cache.exists(key)) else: key = f'{d}|{source}|{asn}|{ipversion}' - self.ranking_cache_wrapper(key) + self._ranking_cache_wrapper(key) r = self.cache.get(key) else: key = f'{d}|asns|{ipversion}' - self.ranking_cache_wrapper(key) + self._ranking_cache_wrapper(key) r = self.cache.zscore(key, asn) - if r: - return float(r) - return 0 + if not r: + r = 0 + if with_position and not source: + to_return['response'] = {'rank': float(r), 'position': self.cache.zrank(key, asn), + 'total_known_asns': self.cache.zcard(key)} + else: + to_return['response'] = float(r) + return to_return def get_sources(self, date: Dates=datetime.date.today()): '''Get the sources availables for a specific day (default: today).''' + to_return = {'meta': {}, 'response': set()} + d = self.__normalize_date(date) + to_return['meta']['date'] = d key = f'{d}|sources' - return self.storage.smembers(key) + to_return['response'] = self.storage.smembers(key) + return to_return def get_asn_descriptions(self, asn: int, all_descriptions=False): + to_return = {'meta': {'asn': asn, 'all_descriptions': all_descriptions}, + 'response': []} descriptions = self.asn_meta.hgetall(f'{asn}|descriptions') if all_descriptions or not descriptions: - return descriptions - return descriptions[sorted(descriptions.keys(), reverse=True)[0]] + to_return['response'] = descriptions + else: + to_return['response'] = descriptions[sorted(descriptions.keys(), reverse=True)[0]] + return to_return def get_prefix_ips(self, asn: int, prefix: str, date: Dates=datetime.date.today(), source: Union[list, str]='', ipversion: str='v4'): + to_return = {'meta': {'asn': asn, 'prefix': prefix, 'ipversion': ipversion, + 'source': source}, + 'response': defaultdict(list)} + + d = self.__normalize_date(date) + to_return['meta']['date'] = d + if source: + to_return['meta']['source'] = source if isinstance(source, list): sources = source else: sources = [source] else: - sources = self.get_sources(date) - prefix_ips = defaultdict(list) - d = self.__normalize_date(date) + sources = self.get_sources(d)['response'] + for source in sources: ips = set([ip_ts.split('|')[0] for ip_ts in self.storage.smembers(f'{d}|{source}|{asn}|{prefix}')]) - [prefix_ips[ip].append(source) for ip in ips] - return prefix_ips + [to_return['response'][ip].append(source) for ip in ips] + return to_return def get_asn_history(self, asn: int, period: int=100, source: Union[list, str]='', ipversion: str='v4', date: Dates=datetime.date.today()): - to_return = [] + to_return = {'meta': {'asn': asn, 'period': period, 'ipversion': ipversion, + 'source': source}, + 'response': []} if isinstance(date, str): date = parse(date).date() if date + timedelta(days=period / 3) > datetime.date.today(): # the period to display will be around the date passed at least 2/3 before the date, at most 1/3 after + # FIXME: That is not doing what it is supposed to... date = datetime.date.today() + to_return['meta']['date'] = date.isoformat() + for i in range(period): d = date - timedelta(days=i) rank = self.asn_rank(asn, d, source, ipversion) - if rank is None: + if 'response' not in rank: rank = 0 - to_return.insert(0, (d.isoformat(), rank)) + to_return['response'].insert(0, (d.isoformat(), rank['response'])) return to_return def country_rank(self, country: str, date: Dates=datetime.date.today(), source: Union[list, str]='', ipversion: str='v4'): - ripe = StatsRIPE() + to_return = {'meta': {'country': country, 'ipversion': ipversion, + 'source': source}, + 'response': []} + d = self.__normalize_date(date) + to_return['meta']['date'] = d + + ripe = StatsRIPE() response = ripe.country_asns(country, query_time=d, details=1) if (not response.get('data') or not response['data'].get('countries') or not response['data']['countries'][0].get('routed')): @@ -175,14 +222,17 @@ class Querying(): # FIXME: return something return 0, [(0, 0)] routed_asns = response['data']['countries'][0]['routed'] - ranks = [self.asn_rank(asn, d, source, ipversion) for asn in routed_asns] - to_return = zip(routed_asns, ranks) - daily_sum = sum(ranks) - return daily_sum, to_return + ranks = [self.asn_rank(asn, d, source, ipversion)['response'] for asn in routed_asns] + print(ranks) + to_return['response'] = [sum(ranks), zip(routed_asns, ranks)] + return to_return def country_history(self, country: Union[list, str], period: int=30, source: Union[list, str]='', ipversion: str='v4', date: Dates=datetime.date.today()): to_return = {} + to_return = {'meta': {'country': country, 'ipversion': ipversion, + 'source': source}, + 'response': defaultdict(list)} if isinstance(date, str): date = parse(date).date() @@ -193,13 +243,12 @@ class Querying(): if isinstance(country, str): country = [country] for c in country: - to_return[c] = [] for i in range(period): d = date - timedelta(days=i) - rank, details = self.country_rank(c, d, source, ipversion) + rank, details = self.country_rank(c, d, source, ipversion)['response'] if rank is None: rank = 0 - to_return[c].insert(0, (d.isoformat(), rank, list(details))) + to_return['response'][c].insert(0, (d.isoformat(), rank, list(details))) return to_return def get_source_config(self): diff --git a/client/bin/bgpranking b/client/bin/bgpranking new file mode 100755 index 0000000..3035299 --- /dev/null +++ b/client/bin/bgpranking @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +try: + import simplejson as json +except ImportError: + import json + +from urllib.parse import urljoin +from pybgpranking import BGPRanking +from pyipasnhistory import IPASNHistory +from datetime import date, timedelta + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Run a query against BGP Ranking') + parser.add_argument('--url', type=str, help='URL of the instance.') + group = parser.add_mutually_exclusive_group(required=True) + + group.add_argument('--asn', help='ASN to lookup') + group.add_argument('--ip', help='IP to lookup') + + args = parser.parse_args() + + if args.url: + bgpranking = BGPRanking(args.url) + ipasn = IPASNHistory(urljoin(args.url, 'ipasn_history')) + else: + bgpranking = BGPRanking() + ipasn = IPASNHistory() + + if args.ip: + response = ipasn.query(args.ip) + print(json.dumps(response, indent=2)) + if 'response' in response and response['response']: + asn = response['response'][list(response['response'].keys())[0]]['asn'] + else: + asn = args.asn + + response = bgpranking.query(asn, date=(date.today() - timedelta(1)).isoformat()) + print(json.dumps(response, indent=2)) diff --git a/client/pybgpranking/__init__.py b/client/pybgpranking/__init__.py new file mode 100644 index 0000000..2ad6888 --- /dev/null +++ b/client/pybgpranking/__init__.py @@ -0,0 +1 @@ +from .api import BGPRanking # noqa diff --git a/client/pybgpranking/api.py b/client/pybgpranking/api.py new file mode 100644 index 0000000..31a6fe9 --- /dev/null +++ b/client/pybgpranking/api.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +try: + import simplejson as json +except ImportError: + import json + +from typing import Union + +import requests +from urllib.parse import urljoin + + +class BGPRanking(): + + def __init__(self, root_url: str='https://bgpranking-ng.circl.lu/'): + self.root_url = root_url + if not self.root_url.endswith('/'): + self.root_url += '/' + self.session = requests.session() + + @property + def is_up(self): + r = self.session.head(self.root_url) + return r.status_code == 200 + + def query(self, asn: str, address_family: str='v4', date: str=None, + source: Union[list, str]=''): + '''Launch a query. + :param asn: ASN to lookup + :param address_family: v4 or v6 + :param date: Exact date to lookup. Fallback to most recent available. + :param source: Source to query. Can be a list of sources. + ''' + to_query = {'asn': asn, 'address_family': address_family} + if date: + to_query['date'] = date + if source: + to_query['source'] = source + r = self.session.post(urljoin(self.root_url, '/json/asn'), data=json.dumps(to_query)) + return r.json() diff --git a/client/requirements.txt b/client/requirements.txt new file mode 100644 index 0000000..f229360 --- /dev/null +++ b/client/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/client/setup.py b/client/setup.py new file mode 100644 index 0000000..90caf85 --- /dev/null +++ b/client/setup.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from setuptools import setup + + +setup( + name='pybgpranking', + version='0.1', + author='Raphaël Vinot', + author_email='raphael.vinot@circl.lu', + maintainer='Raphaël Vinot', + url='https://github.com/D4-project/BGP-Ranking/client', + description='Python client for BGP Ranking', + packages=['pybgpranking'], + scripts=['bin/bgpranking'], + classifiers=[ + 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', + 'Development Status :: 3 - Alpha', + 'Environment :: Console', + 'Operating System :: POSIX :: Linux', + 'Intended Audience :: Science/Research', + 'Intended Audience :: Telecommunications Industry', + 'Intended Audience :: Information Technology', + 'Programming Language :: Python :: 3', + 'Topic :: Security', + 'Topic :: Internet', + ] +) diff --git a/website/web/__init__.py b/website/web/__init__.py index 0171dd4..d21f5c4 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -27,6 +27,8 @@ Bootstrap(app) app.config['BOOTSTRAP_SERVE_LOCAL'] = True +# ############# Helpers ############# + def get_request_parameter(parameter): if request.method == 'POST': d = request.form @@ -72,20 +74,10 @@ def get_country_codes(): for c in pycountry.countries: yield c.alpha_2, c.name +# ############# Helpers ###################### -@app.route('/ipasn_history/', defaults={'path': ''}, methods=['GET', 'POST']) -@app.route('/ipasn_history/', methods=['GET', 'POST']) -def ipasn_history_proxy(path): - config, general_config_file = load_general_config() - if 'ipasnhistory_url' not in config: - raise MissingConfigEntry(f'"ipasnhistory_url" is missing in {general_config_file}.') - proxied_url = urljoin(config['ipasnhistory_url'], request.full_path.replace('/ipasn_history', '')) - if request.method in ['GET', 'HEAD']: - to_return = requests.get(proxied_url).json() - elif request.method == 'POST': - to_return = requests.post(proxied_url, data=request.data).json() - return Response(json.dumps(to_return), mimetype='application/json') +# ############# Web UI ############# @app.route('/', methods=['GET', 'POST']) def index(): @@ -94,12 +86,11 @@ def index(): return 'Ack' load_session() q = Querying() - sources = q.get_sources(date=session['date']) + sources = q.get_sources(date=session['date'])['response'] session.pop('asn', None) session.pop('country', None) - ranks = q.asns_global_ranking(limit=100, **session) - descriptions = [q.get_asn_descriptions(int(asn)) for asn, rank in ranks] - r = zip(ranks, descriptions) + ranks = q.asns_global_ranking(limit=100, **session)['response'] + r = [(asn, rank, q.get_asn_descriptions(int(asn))['response']) for asn, rank in ranks] return render_template('index.html', ranks=r, sources=sources, countries=get_country_codes(), **session) @@ -109,47 +100,31 @@ def asn_details(): q = Querying() if 'asn' not in session: return redirect(url_for('/')) - asn_descriptions = q.get_asn_descriptions(asn=session['asn'], all_descriptions=True) - sources = q.get_sources(date=session['date']) - ranks = q.asn_details(**session) + asn_descriptions = q.get_asn_descriptions(asn=session['asn'], all_descriptions=True)['response'] + sources = q.get_sources(date=session['date'])['response'] + ranks = q.asn_details(**session)['response'] prefix = get_request_parameter('prefix') if prefix: - prefix_ips = q.get_prefix_ips(prefix=prefix, **session) + prefix_ips = q.get_prefix_ips(prefix=prefix, **session)['response'] prefix_ips = [(ip, sorted(sources)) for ip, sources in prefix_ips.items()] prefix_ips.sort(key=lambda entry: len(entry[1]), reverse=True) else: prefix_ips = [] - return render_template('asn.html', sources=sources, ranks=ranks, prefix_ips=prefix_ips, asn_descriptions=asn_descriptions, **session) + return render_template('asn.html', sources=sources, ranks=ranks, + prefix_ips=prefix_ips, asn_descriptions=asn_descriptions, **session) -@app.route('/asn_description', methods=['POST']) -def asn_description(): - load_session() - asn = None - if request.form.get('asn'): - asn = request.form.get('asn') - elif session.get('asn'): - asn = session.get('asn') - else: - to_return = {'error': 'asn required'} - if asn: - q = Querying() - to_return = q.get_asn_descriptions(asn, session.get('all_descriptions')) - return Response(json.dumps(to_return), mimetype='application/json') - - -@app.route('/asn_history', methods=['GET', 'POST']) -def asn_history(): +@app.route('/country', methods=['GET', 'POST']) +def country(): load_session() q = Querying() - if 'asn' in session: - return Response(json.dumps(q.get_asn_history(**session)), mimetype='application/json') - return Response(json.dumps({'error': f'asn key is required: {session}'}), mimetype='application/json') + sources = q.get_sources(date=session['date'])['response'] + return render_template('country.html', sources=sources, countries=get_country_codes(), **session) @app.route('/country_history_callback', methods=['GET', 'POST']) def country_history_callback(): - history_data = json.loads(request.data) + history_data = request.get_json(force=True) to_display = [] mapping = defaultdict(dict) dates = [] @@ -172,19 +147,86 @@ def country_history_callback(): line.append('N/A') to_display_temp.append(line) to_display.append(to_display_temp) - return json.dumps(render_template('country_asn_map.html', to_display=to_display)) + return render_template('country_asn_map.html', to_display=to_display) + +# ############# Web UI ############# -@app.route('/country_history', methods=['GET', 'POST']) +# ############# Json outputs ############# + +@app.route('/ipasn_history/', defaults={'path': ''}, methods=['GET', 'POST']) +@app.route('/ipasn_history/', methods=['GET', 'POST']) +def ipasn_history_proxy(path): + config, general_config_file = load_general_config() + if 'ipasnhistory_url' not in config: + raise MissingConfigEntry(f'"ipasnhistory_url" is missing in {general_config_file}.') + proxied_url = urljoin(config['ipasnhistory_url'], request.full_path.replace('/ipasn_history', '')) + if request.method in ['GET', 'HEAD']: + to_return = requests.get(proxied_url).json() + elif request.method == 'POST': + to_return = requests.post(proxied_url, data=request.data).json() + return Response(json.dumps(to_return), mimetype='application/json') + + +@app.route('/json/asn', methods=['POST']) +def json_asn(): + # TODO + # * Filter on date => if only returning one descr, return the desription at that date + query = request.get_json(force=True) + to_return = {'meta': query, 'response': {}} + if 'asn' not in query: + to_return['error'] = f'You need to pass an asn - {query}' + return to_return + + q = Querying() + asn_description_query = {'asn': query['asn']} + if 'all_descriptions' in query: + asn_description_query['all_descriptions'] = query['all_descriptions'] + to_return['response']['asn_description'] = q.get_asn_descriptions(**asn_description_query)['response'] + + asn_rank_query = {'asn': query['asn']} + if 'date' in query: + asn_rank_query['date'] = query['date'] + if 'source' in query: + asn_rank_query['source'] = query['source'] + else: + asn_rank_query['with_position'] = True + if 'ipversion' in query: + asn_rank_query['ipversion'] = query['ipversion'] + + to_return['response']['ranking'] = q.asn_rank(**asn_rank_query)['response'] + return Response(json.dumps(to_return), mimetype='application/json') + + +@app.route('/json/asn_description', methods=['POST']) +def asn_description(): + load_session() + asn = None + if request.form.get('asn'): + asn = request.form.get('asn') + elif session.get('asn'): + asn = session.get('asn') + else: + to_return = {'error': 'asn required'} + if asn: + q = Querying() + to_return = q.get_asn_descriptions(asn, session.get('all_descriptions')) + return Response(json.dumps(to_return), mimetype='application/json') + + +@app.route('/json/asn_history', methods=['GET', 'POST']) +def asn_history(): + load_session() + q = Querying() + if 'asn' in session: + return Response(json.dumps(q.get_asn_history(**session)), mimetype='application/json') + return Response(json.dumps({'error': f'asn key is required: {session}'}), mimetype='application/json') + + +@app.route('/json/country_history', methods=['GET', 'POST']) def country_history(): load_session() q = Querying() return Response(json.dumps(q.country_history(**session)), mimetype='application/json') - -@app.route('/country', methods=['GET', 'POST']) -def country(): - load_session() - q = Querying() - sources = q.get_sources(date=session['date']) - return render_template('country.html', sources=sources, countries=get_country_codes(), **session) +# ############# Json outputs ############# diff --git a/website/web/static/linegraph.js b/website/web/static/linegraph.js index 57a0867..7279e11 100644 --- a/website/web/static/linegraph.js +++ b/website/web/static/linegraph.js @@ -25,14 +25,14 @@ function linegraph(call_path) { // Get the data d3.json(call_path, {credentials: 'same-origin'}).then(function(data) { - x.domain(d3.extent(data, function(d) { return parseTime(d[0]); })); - y.domain(d3.extent(data, function(d) { return d[1]; })); + x.domain(d3.extent(data.response, function(d) { return parseTime(d[0]); })); + y.domain(d3.extent(data.response, function(d) { return d[1]; })); xAxis(); yAxis(); context.beginPath(); - line(data); + line(data.response); context.lineWidth = 1.5; context.strokeStyle = "steelblue"; context.stroke(); diff --git a/website/web/static/linegraph_country.js b/website/web/static/linegraph_country.js index fff4a32..6acc867 100644 --- a/website/web/static/linegraph_country.js +++ b/website/web/static/linegraph_country.js @@ -18,7 +18,7 @@ function linegraph(call_path) { d3.json(call_path, {credentials: 'same-origin'}).then(function(data) { - var country_ranks = d3.entries(data).map(function(country_rank) { + var country_ranks = d3.entries(data.response).map(function(country_rank) { return { country: country_rank.key, values: d3.values(country_rank.value).map(function(d) { @@ -68,11 +68,10 @@ function linegraph(call_path) { .style("font", "10px sans-serif") .text(function(d) { return d.id; }); - d3.json(call_path + '_callback', + d3.text('/country_history_callback', {credentials: 'same-origin', method: 'POST', - body: JSON.stringify(data), - // headers: {'Content-Type': 'application/json'} + body: JSON.stringify(data.response), }).then(function(data) { d3.select('#asn_details').html(data); }); diff --git a/website/web/templates/asn.html b/website/web/templates/asn.html index a352f14..2bec7df 100644 --- a/website/web/templates/asn.html +++ b/website/web/templates/asn.html @@ -12,7 +12,7 @@ {% block scripts %} {{ super() }} - + {% endblock %} {% block content %} diff --git a/website/web/templates/country.html b/website/web/templates/country.html index 069e422..bf1fdad 100644 --- a/website/web/templates/country.html +++ b/website/web/templates/country.html @@ -11,7 +11,7 @@ {% block scripts %} {{ super() }} - + {% endblock %} {% block content %} diff --git a/website/web/templates/index.html b/website/web/templates/index.html index 2b7a608..27e00b4 100644 --- a/website/web/templates/index.html +++ b/website/web/templates/index.html @@ -18,7 +18,7 @@ Rank Description - {% for (asn, rank), description in ranks %} + {% for asn, rank, description in ranks %} {{ asn }} {{ rank }}