diff --git a/.gitignore b/.gitignore index 123d5a4..542018f 100644 --- a/.gitignore +++ b/.gitignore @@ -120,5 +120,9 @@ rawdata storage/db/ storage/kvrocks* -website/web/static/d3.v5.js +ranking/db/ +ranking/kvrocks* +website/web/static/d3.*.js website/web/static/bootstrap-select.min.* + +*.pid diff --git a/website/web/__init__.py b/website/web/__init__.py index 22b6283..eccb851 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -1,27 +1,21 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import json import pkg_resources from collections import defaultdict from datetime import date, timedelta -from urllib.parse import urljoin -from typing import Dict, Any, Tuple, List, Optional, Union +from typing import Dict, Any, Tuple, List, Optional -import pycountry # type: ignore -import requests - -from flask import Flask, render_template, request, session, Response, redirect, url_for +from flask import Flask, render_template, request, session, redirect, url_for from flask_bootstrap import Bootstrap # type: ignore from flask_restx import Api # type: ignore from bgpranking.bgpranking import BGPRanking -from bgpranking.default import get_config from bgpranking.helpers import get_ipasn from .genericapi import api as generic_api -from .helpers import get_secret_key +from .helpers import get_secret_key, load_session, get_country_codes from .proxied import ReverseProxied app = Flask(__name__) @@ -36,41 +30,6 @@ app.config['BOOTSTRAP_SERVE_LOCAL'] = True bgpranking = BGPRanking() -# ############# Helpers ############# - -def load_session(): - if request.method == 'POST': - d = request.form - elif request.method == 'GET': - d = request.args # type: ignore - - for key in d: - if '_all' in d.getlist(key): - session.pop(key, None) - else: - values = [v for v in d.getlist(key) if v] - if values: - if len(values) == 1: - session[key] = values[0] - else: - session[key] = values - - # Edge cases - if 'asn' in session: - session.pop('country', None) - elif 'country' in session: - session.pop('asn', None) - if 'date' not in session: - session['date'] = (date.today() - timedelta(days=1)).isoformat() - - -def get_country_codes(): - for c in pycountry.countries: - yield c.alpha_2, c.name - -# ############# Helpers ###################### - - # ############# Web UI ############# @app.route('/', methods=['GET', 'POST']) @@ -176,109 +135,11 @@ def ipasn(): # ############# Web UI ############# - -# ############# Json outputs ############# - -@app.route('/ipasn_history/', defaults={'path': ''}, methods=['GET', 'POST']) -@app.route('/ipasn_history/', methods=['GET', 'POST']) -def ipasn_history_proxy(path): - - path_for_ipasnhistory = request.full_path.replace('/ipasn_history', '') - if '/?' in path_for_ipasnhistory: - path_for_ipasnhistory = path_for_ipasnhistory.replace('/?', '/ip?') - proxied_url = urljoin(get_config('generic', 'ipasnhistory_url'), path_for_ipasnhistory) - if request.method in ['GET', 'HEAD']: - to_return = requests.get(proxied_url).json() - elif request.method == 'POST': - to_return = requests.post(proxied_url, data=request.data).json() - return Response(json.dumps(to_return), mimetype='application/json') - - -@app.route('/json/asn', methods=['POST']) -def json_asn(): - # TODO - # * Filter on date => if only returning one descr, return the desription at that date - query: Dict[str, Any] = request.get_json(force=True) # type: ignore - to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}} - if 'asn' not in query: - to_return['error'] = f'You need to pass an asn - {query}' - return Response(json.dumps(to_return), mimetype='application/json') - - asn_description_query = {'asn': query['asn']} - if 'all_descriptions' in query: - asn_description_query['all_descriptions'] = query['all_descriptions'] - responses = bgpranking.get_asn_descriptions(**asn_description_query)['response'] - to_return['response']['asn_description'] = responses # type: ignore - - asn_rank_query = {'asn': query['asn']} - if 'date' in query: - asn_rank_query['date'] = query['date'] - if 'source' in query: - asn_rank_query['source'] = query['source'] - else: - asn_rank_query['with_position'] = True - if 'ipversion' in query: - asn_rank_query['ipversion'] = query['ipversion'] - - to_return['response']['ranking'] = bgpranking.asn_rank(**asn_rank_query)['response'] # type: ignore - return Response(json.dumps(to_return), mimetype='application/json') - - -@app.route('/json/asn_descriptions', methods=['POST']) -def asn_description(): - query: Dict = request.get_json(force=True) # type: ignore - to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}} - if 'asn' not in query: - to_return['error'] = f'You need to pass an asn - {query}' - return Response(json.dumps(to_return), mimetype='application/json') - - to_return['response']['asn_descriptions'] = bgpranking.get_asn_descriptions(**query)['response'] # type: ignore - return Response(json.dumps(to_return), mimetype='application/json') - - -@app.route('/json/asn_history', methods=['GET', 'POST']) -def asn_history(): - if request.method == 'GET': - load_session() - if 'asn' in session: - return Response(json.dumps(bgpranking.get_asn_history(**session)), mimetype='application/json') - - query: Dict = request.get_json(force=True) # type: ignore - to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}} - if 'asn' not in query: - to_return['error'] = f'You need to pass an asn - {query}' - return Response(json.dumps(to_return), mimetype='application/json') - - to_return['response']['asn_history'] = bgpranking.get_asn_history(**query)['response'] # type: ignore - return Response(json.dumps(to_return), mimetype='application/json') - - -@app.route('/json/country_history', methods=['GET', 'POST']) -def country_history(): - if request.method == 'GET': - load_session() - return Response(json.dumps(bgpranking.country_history(**session)), mimetype='application/json') - - query: Dict = request.get_json(force=True) # type: ignore - to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}} - to_return['response']['country_history'] = bgpranking.country_history(**query)['response'] # type: ignore - return Response(json.dumps(to_return), mimetype='application/json') - - -@app.route('/json/asns_global_ranking', methods=['POST']) -def json_asns_global_ranking(): - query: Dict = request.get_json(force=True) # type: ignore - to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}} - to_return['response'] = bgpranking.asns_global_ranking(**query)['response'] - return Response(json.dumps(to_return), mimetype='application/json') - -# ############# Json outputs ############# - - # Query API api = Api(app, title='BGP Ranking API', description='API to query BGP Ranking.', + doc='/doc/', version=pkg_resources.get_distribution('bgpranking').version) api.add_namespace(generic_api) diff --git a/website/web/genericapi.py b/website/web/genericapi.py index 5145375..abe9b91 100644 --- a/website/web/genericapi.py +++ b/website/web/genericapi.py @@ -1,25 +1,20 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import pkg_resources +from typing import Dict, Any, Union +from urllib.parse import urljoin -from flask import Flask -from flask_restx import Api, Resource # type: ignore +import requests +from flask import request, session +from flask_restx import Namespace, Resource, fields # type: ignore + +from bgpranking.default import get_config from bgpranking.bgpranking import BGPRanking -from .helpers import get_secret_key -from .proxied import ReverseProxied +from .helpers import load_session -app: Flask = Flask(__name__) - -app.wsgi_app = ReverseProxied(app.wsgi_app) # type: ignore - -app.config['SECRET_KEY'] = get_secret_key() - -api = Api(app, title='BGP Ranking API', - description='API to query BGP Ranking.', - version=pkg_resources.get_distribution('bgpranking').version) +api = Namespace('BGP Ranking API', description='API to query BGP Ranking.', path='/') bgpranking: BGPRanking = BGPRanking() @@ -30,3 +25,149 @@ class RedisUp(Resource): def get(self): return bgpranking.check_redis_up() + + +@api.route('/ipasn_history/') +@api.route('/ipasn_history/') +class IPASNProxy(Resource): + + def _proxy_url(self): + if request.full_path[-1] == '?': + full_path = request.full_path[:-1] + else: + full_path = request.full_path + path_for_ipasnhistory = full_path.replace('/ipasn_history', '') + if path_for_ipasnhistory.startswith('/?'): + path_for_ipasnhistory = path_for_ipasnhistory.replace('/?', '/ip?') + return urljoin(get_config('generic', 'ipasnhistory_url'), path_for_ipasnhistory) + + def get(self, path=''): + url = self._proxy_url() + print(url) + return requests.get(url).json() + + def post(self, path=''): + url = self._proxy_url() + return requests.post(url, data=request.data).json() + + +# TODO: Add other parameters for asn_rank +asn_query_fields = api.model('ASNQueryFields', { + 'asn': fields.String(description='The Autonomus System Number to search', required=True) +}) + + +@api.route('/json/asn') +class ASNRank(Resource): + + @api.doc(body=asn_query_fields) + def post(self): + # TODO + # * Filter on date => if only returning one descr, return the desription at that date + query: Dict[str, Any] = request.get_json(force=True) # type: ignore + to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}} + if 'asn' not in query: + to_return['error'] = f'You need to pass an asn - {query}' + return to_return + + asn_description_query = {'asn': query['asn']} + responses = bgpranking.get_asn_descriptions(**asn_description_query)['response'] + to_return['response']['asn_description'] = responses # type: ignore + + asn_rank_query = {'asn': query['asn']} + if 'date' in query: + asn_rank_query['date'] = query['date'] + if 'source' in query: + asn_rank_query['source'] = query['source'] + else: + asn_rank_query['with_position'] = True + if 'ipversion' in query: + asn_rank_query['ipversion'] = query['ipversion'] + + to_return['response']['ranking'] = bgpranking.asn_rank(**asn_rank_query)['response'] # type: ignore + return to_return + + +asn_descr_fields = api.model('ASNDescriptionsFields', { + 'asn': fields.String(description='The Autonomus System Number to search', required=True), + 'all_descriptions': fields.Boolean(description='If true, returns all the descriptions instead of only the last one', default=False) +}) + + +@api.route('/json/asn_descriptions') +class ASNDescription(Resource): + + @api.doc(body=asn_descr_fields) + def post(self): + query: Dict = request.get_json(force=True) # type: ignore + to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}} + if 'asn' not in query: + to_return['error'] = f'You need to pass an asn - {query}' + return to_return + + to_return['response']['asn_descriptions'] = bgpranking.get_asn_descriptions(**query)['response'] # type: ignore + return to_return + + +# TODO: Add other parameters for get_asn_history +asn_history_fields = api.model('ASNQueryFields', { + 'asn': fields.String(description='The Autonomus System Number to search', required=True) +}) + + +@api.route('/json/asn_history') +class ASNHistory(Resource): + + def get(self): + load_session() + if 'asn' in session: + return bgpranking.get_asn_history(**session) + + @api.doc(body=asn_history_fields) + def post(self): + query: Dict = request.get_json(force=True) # type: ignore + to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}} + if 'asn' not in query: + to_return['error'] = f'You need to pass an asn - {query}' + return to_return + + to_return['response']['asn_history'] = bgpranking.get_asn_history(**query)['response'] # type: ignore + return to_return + + +# TODO: Add other parameters for country_history +coutry_history_fields = api.model('CountryHistoryFields', { + 'country': fields.String(description='The Country Code', required=True) +}) + + +@api.route('/json/country_history') +class CountryHistory(Resource): + + def get(self): + load_session() + return bgpranking.country_history(**session) + + @api.doc(body=coutry_history_fields) + def post(self): + query: Dict = request.get_json(force=True) # type: ignore + to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}} + to_return['response']['country_history'] = bgpranking.country_history(**query)['response'] # type: ignore + return to_return + + +# TODO: Add other parameters for asns_global_ranking +asns_global_ranking_fields = api.model('ASNsGlobalRankingFields', { + 'date': fields.String(description='The date') +}) + + +@api.route('/json/asns_global_ranking') +class ASNsGlobalRanking(Resource): + + @api.doc(body=asns_global_ranking_fields) + def post(self): + query: Dict = request.get_json(force=True) # type: ignore + to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}} + to_return['response'] = bgpranking.asns_global_ranking(**query)['response'] + return to_return diff --git a/website/web/helpers.py b/website/web/helpers.py index f5fcb3c..730b3eb 100644 --- a/website/web/helpers.py +++ b/website/web/helpers.py @@ -2,9 +2,15 @@ # -*- coding: utf-8 -*- import os + +from datetime import date, timedelta from functools import lru_cache from pathlib import Path +import pycountry # type: ignore + +from flask import request, session + from bgpranking.default import get_homedir @@ -25,3 +31,34 @@ def get_secret_key() -> bytes: f.write(os.urandom(64)) with secret_file_path.open('rb') as f: return f.read() + + +def load_session(): + if request.method == 'POST': + d = request.form + elif request.method == 'GET': + d = request.args # type: ignore + + for key in d: + if '_all' in d.getlist(key): + session.pop(key, None) + else: + values = [v for v in d.getlist(key) if v] + if values: + if len(values) == 1: + session[key] = values[0] + else: + session[key] = values + + # Edge cases + if 'asn' in session: + session.pop('country', None) + elif 'country' in session: + session.pop('asn', None) + if 'date' not in session: + session['date'] = (date.today() - timedelta(days=1)).isoformat() + + +def get_country_codes(): + for c in pycountry.countries: + yield c.alpha_2, c.name