chg: Move API to restx

.gitignore

@@ -120,5 +120,9 @@ rawdata
 
 storage/db/
 storage/kvrocks*
-website/web/static/d3.v5.js
+ranking/db/
+ranking/kvrocks*
+website/web/static/d3.*.js
 website/web/static/bootstrap-select.min.*
+
+*.pid

website/web/__init__.py

@@ -1,27 +1,21 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-import json
 import pkg_resources
 
 from collections import defaultdict
 from datetime import date, timedelta
-from urllib.parse import urljoin
+from typing import Dict, Any, Tuple, List, Optional
-from typing import Dict, Any, Tuple, List, Optional, Union
 
-import pycountry  # type: ignore
+from flask import Flask, render_template, request, session, redirect, url_for
-import requests
-
-from flask import Flask, render_template, request, session, Response, redirect, url_for
 from flask_bootstrap import Bootstrap  # type: ignore
 from flask_restx import Api  # type: ignore
 
 from bgpranking.bgpranking import BGPRanking
-from bgpranking.default import get_config
 from bgpranking.helpers import get_ipasn
 
 from .genericapi import api as generic_api
-from .helpers import get_secret_key
+from .helpers import get_secret_key, load_session, get_country_codes
 from .proxied import ReverseProxied
 
 app = Flask(__name__)
@@ -36,41 +30,6 @@ app.config['BOOTSTRAP_SERVE_LOCAL'] = True
 bgpranking = BGPRanking()
 
 
-# ############# Helpers #############
-
-def load_session():
-    if request.method == 'POST':
-        d = request.form
-    elif request.method == 'GET':
-        d = request.args  # type: ignore
-
-    for key in d:
-        if '_all' in d.getlist(key):
-            session.pop(key, None)
-        else:
-            values = [v for v in d.getlist(key) if v]
-            if values:
-                if len(values) == 1:
-                    session[key] = values[0]
-                else:
-                    session[key] = values
-
-    # Edge cases
-    if 'asn' in session:
-        session.pop('country', None)
-    elif 'country' in session:
-        session.pop('asn', None)
-    if 'date' not in session:
-        session['date'] = (date.today() - timedelta(days=1)).isoformat()
-
-
-def get_country_codes():
-    for c in pycountry.countries:
-        yield c.alpha_2, c.name
-
-# ############# Helpers ######################
-
-
 # ############# Web UI #############
 
 @app.route('/', methods=['GET', 'POST'])
@@ -176,109 +135,11 @@ def ipasn():
 
 # ############# Web UI #############
 
-
-# ############# Json outputs #############
-
-@app.route('/ipasn_history/', defaults={'path': ''}, methods=['GET', 'POST'])
-@app.route('/ipasn_history/<path:path>', methods=['GET', 'POST'])
-def ipasn_history_proxy(path):
-
-    path_for_ipasnhistory = request.full_path.replace('/ipasn_history', '')
-    if '/?' in path_for_ipasnhistory:
-        path_for_ipasnhistory = path_for_ipasnhistory.replace('/?', '/ip?')
-    proxied_url = urljoin(get_config('generic', 'ipasnhistory_url'), path_for_ipasnhistory)
-    if request.method in ['GET', 'HEAD']:
-        to_return = requests.get(proxied_url).json()
-    elif request.method == 'POST':
-        to_return = requests.post(proxied_url, data=request.data).json()
-    return Response(json.dumps(to_return), mimetype='application/json')
-
-
-@app.route('/json/asn', methods=['POST'])
-def json_asn():
-    # TODO
-    # * Filter on date => if only returning one descr, return the desription at that date
-    query: Dict[str, Any] = request.get_json(force=True)  # type: ignore
-    to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
-    if 'asn' not in query:
-        to_return['error'] = f'You need to pass an asn - {query}'
-        return Response(json.dumps(to_return), mimetype='application/json')
-
-    asn_description_query = {'asn': query['asn']}
-    if 'all_descriptions' in query:
-        asn_description_query['all_descriptions'] = query['all_descriptions']
-    responses = bgpranking.get_asn_descriptions(**asn_description_query)['response']
-    to_return['response']['asn_description'] = responses  # type: ignore
-
-    asn_rank_query = {'asn': query['asn']}
-    if 'date' in query:
-        asn_rank_query['date'] = query['date']
-    if 'source' in query:
-        asn_rank_query['source'] = query['source']
-    else:
-        asn_rank_query['with_position'] = True
-    if 'ipversion' in query:
-        asn_rank_query['ipversion'] = query['ipversion']
-
-    to_return['response']['ranking'] = bgpranking.asn_rank(**asn_rank_query)['response']  # type: ignore
-    return Response(json.dumps(to_return), mimetype='application/json')
-
-
-@app.route('/json/asn_descriptions', methods=['POST'])
-def asn_description():
-    query: Dict = request.get_json(force=True)  # type: ignore
-    to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
-    if 'asn' not in query:
-        to_return['error'] = f'You need to pass an asn - {query}'
-        return Response(json.dumps(to_return), mimetype='application/json')
-
-    to_return['response']['asn_descriptions'] = bgpranking.get_asn_descriptions(**query)['response']  # type: ignore
-    return Response(json.dumps(to_return), mimetype='application/json')
-
-
-@app.route('/json/asn_history', methods=['GET', 'POST'])
-def asn_history():
-    if request.method == 'GET':
-        load_session()
-        if 'asn' in session:
-            return Response(json.dumps(bgpranking.get_asn_history(**session)), mimetype='application/json')
-
-    query: Dict = request.get_json(force=True)  # type: ignore
-    to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
-    if 'asn' not in query:
-        to_return['error'] = f'You need to pass an asn - {query}'
-        return Response(json.dumps(to_return), mimetype='application/json')
-
-    to_return['response']['asn_history'] = bgpranking.get_asn_history(**query)['response']  # type: ignore
-    return Response(json.dumps(to_return), mimetype='application/json')
-
-
-@app.route('/json/country_history', methods=['GET', 'POST'])
-def country_history():
-    if request.method == 'GET':
-        load_session()
-        return Response(json.dumps(bgpranking.country_history(**session)), mimetype='application/json')
-
-    query: Dict = request.get_json(force=True)  # type: ignore
-    to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
-    to_return['response']['country_history'] = bgpranking.country_history(**query)['response']  # type: ignore
-    return Response(json.dumps(to_return), mimetype='application/json')
-
-
-@app.route('/json/asns_global_ranking', methods=['POST'])
-def json_asns_global_ranking():
-    query: Dict = request.get_json(force=True)  # type: ignore
-    to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
-    to_return['response'] = bgpranking.asns_global_ranking(**query)['response']
-    return Response(json.dumps(to_return), mimetype='application/json')
-
-# ############# Json outputs #############
-
-
 # Query API
 
 api = Api(app, title='BGP Ranking API',
           description='API to query BGP Ranking.',
+          doc='/doc/',
           version=pkg_resources.get_distribution('bgpranking').version)
 
 api.add_namespace(generic_api)

website/web/genericapi.py

@@ -1,25 +1,20 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import pkg_resources
+from typing import Dict, Any, Union
+from urllib.parse import urljoin
 
-from flask import Flask
+import requests
-from flask_restx import Api, Resource  # type: ignore
 
+from flask import request, session
+from flask_restx import Namespace, Resource, fields  # type: ignore
+
+from bgpranking.default import get_config
 from bgpranking.bgpranking import BGPRanking
 
-from .helpers import get_secret_key
+from .helpers import load_session
-from .proxied import ReverseProxied
 
-app: Flask = Flask(__name__)
+api = Namespace('BGP Ranking API', description='API to query BGP Ranking.', path='/')
-
-app.wsgi_app = ReverseProxied(app.wsgi_app)  # type: ignore
-
-app.config['SECRET_KEY'] = get_secret_key()
-
-api = Api(app, title='BGP Ranking API',
-          description='API to query BGP Ranking.',
-          version=pkg_resources.get_distribution('bgpranking').version)
 
 bgpranking: BGPRanking = BGPRanking()
 
@@ -30,3 +25,149 @@ class RedisUp(Resource):
 
     def get(self):
         return bgpranking.check_redis_up()
+
+
+@api.route('/ipasn_history/')
+@api.route('/ipasn_history/<path:path>')
+class IPASNProxy(Resource):
+
+    def _proxy_url(self):
+        if request.full_path[-1] == '?':
+            full_path = request.full_path[:-1]
+        else:
+            full_path = request.full_path
+        path_for_ipasnhistory = full_path.replace('/ipasn_history', '')
+        if path_for_ipasnhistory.startswith('/?'):
+            path_for_ipasnhistory = path_for_ipasnhistory.replace('/?', '/ip?')
+        return urljoin(get_config('generic', 'ipasnhistory_url'), path_for_ipasnhistory)
+
+    def get(self, path=''):
+        url = self._proxy_url()
+        print(url)
+        return requests.get(url).json()
+
+    def post(self, path=''):
+        url = self._proxy_url()
+        return requests.post(url, data=request.data).json()
+
+
+# TODO: Add other parameters for asn_rank
+asn_query_fields = api.model('ASNQueryFields', {
+    'asn': fields.String(description='The Autonomus System Number to search', required=True)
+})
+
+
+@api.route('/json/asn')
+class ASNRank(Resource):
+
+    @api.doc(body=asn_query_fields)
+    def post(self):
+        # TODO
+        # * Filter on date => if only returning one descr, return the desription at that date
+        query: Dict[str, Any] = request.get_json(force=True)  # type: ignore
+        to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
+        if 'asn' not in query:
+            to_return['error'] = f'You need to pass an asn - {query}'
+            return to_return
+
+        asn_description_query = {'asn': query['asn']}
+        responses = bgpranking.get_asn_descriptions(**asn_description_query)['response']
+        to_return['response']['asn_description'] = responses  # type: ignore
+
+        asn_rank_query = {'asn': query['asn']}
+        if 'date' in query:
+            asn_rank_query['date'] = query['date']
+        if 'source' in query:
+            asn_rank_query['source'] = query['source']
+        else:
+            asn_rank_query['with_position'] = True
+        if 'ipversion' in query:
+            asn_rank_query['ipversion'] = query['ipversion']
+
+        to_return['response']['ranking'] = bgpranking.asn_rank(**asn_rank_query)['response']  # type: ignore
+        return to_return
+
+
+asn_descr_fields = api.model('ASNDescriptionsFields', {
+    'asn': fields.String(description='The Autonomus System Number to search', required=True),
+    'all_descriptions': fields.Boolean(description='If true, returns all the descriptions instead of only the last one', default=False)
+})
+
+
+@api.route('/json/asn_descriptions')
+class ASNDescription(Resource):
+
+    @api.doc(body=asn_descr_fields)
+    def post(self):
+        query: Dict = request.get_json(force=True)  # type: ignore
+        to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
+        if 'asn' not in query:
+            to_return['error'] = f'You need to pass an asn - {query}'
+            return to_return
+
+        to_return['response']['asn_descriptions'] = bgpranking.get_asn_descriptions(**query)['response']  # type: ignore
+        return to_return
+
+
+# TODO: Add other parameters for get_asn_history
+asn_history_fields = api.model('ASNQueryFields', {
+    'asn': fields.String(description='The Autonomus System Number to search', required=True)
+})
+
+
+@api.route('/json/asn_history')
+class ASNHistory(Resource):
+
+    def get(self):
+        load_session()
+        if 'asn' in session:
+            return bgpranking.get_asn_history(**session)
+
+    @api.doc(body=asn_history_fields)
+    def post(self):
+        query: Dict = request.get_json(force=True)  # type: ignore
+        to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
+        if 'asn' not in query:
+            to_return['error'] = f'You need to pass an asn - {query}'
+            return to_return
+
+        to_return['response']['asn_history'] = bgpranking.get_asn_history(**query)['response']  # type: ignore
+        return to_return
+
+
+# TODO: Add other parameters for country_history
+coutry_history_fields = api.model('CountryHistoryFields', {
+    'country': fields.String(description='The Country Code', required=True)
+})
+
+
+@api.route('/json/country_history')
+class CountryHistory(Resource):
+
+    def get(self):
+        load_session()
+        return bgpranking.country_history(**session)
+
+    @api.doc(body=coutry_history_fields)
+    def post(self):
+        query: Dict = request.get_json(force=True)  # type: ignore
+        to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
+        to_return['response']['country_history'] = bgpranking.country_history(**query)['response']  # type: ignore
+        return to_return
+
+
+# TODO: Add other parameters for asns_global_ranking
+asns_global_ranking_fields = api.model('ASNsGlobalRankingFields', {
+    'date': fields.String(description='The date')
+})
+
+
+@api.route('/json/asns_global_ranking')
+class ASNsGlobalRanking(Resource):
+
+    @api.doc(body=asns_global_ranking_fields)
+    def post(self):
+        query: Dict = request.get_json(force=True)  # type: ignore
+        to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
+        to_return['response'] = bgpranking.asns_global_ranking(**query)['response']
+        return to_return

website/web/helpers.py

@@ -2,9 +2,15 @@
 # -*- coding: utf-8 -*-
 
 import os
+
+from datetime import date, timedelta
 from functools import lru_cache
 from pathlib import Path
 
+import pycountry  # type: ignore
+
+from flask import request, session
+
 from bgpranking.default import get_homedir
 
 
@@ -25,3 +31,34 @@ def get_secret_key() -> bytes:
                 f.write(os.urandom(64))
     with secret_file_path.open('rb') as f:
         return f.read()
+
+
+def load_session():
+    if request.method == 'POST':
+        d = request.form
+    elif request.method == 'GET':
+        d = request.args  # type: ignore
+
+    for key in d:
+        if '_all' in d.getlist(key):
+            session.pop(key, None)
+        else:
+            values = [v for v in d.getlist(key) if v]
+            if values:
+                if len(values) == 1:
+                    session[key] = values[0]
+                else:
+                    session[key] = values
+
+    # Edge cases
+    if 'asn' in session:
+        session.pop('country', None)
+    elif 'country' in session:
+        session.pop('asn', None)
+    if 'date' not in session:
+        session['date'] = (date.today() - timedelta(days=1)).isoformat()
+
+
+def get_country_codes():
+    for c in pycountry.countries:
+        yield c.alpha_2, c.name