new: Use new BGP Ranking & IPASN History

bin/start.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from subprocess import Popen
+from urlabuse.helpers import get_homedir
+
+import redis
+import sys
+
+if redis.VERSION < (3, ):
+    print('redis-py >= 3 is required.')
+    sys.exit()
+
+if __name__ == '__main__':
+    # Just fail if the env isn't set.
+    get_homedir()
+    p = Popen(['run_backend.py', '--start'])
+    p.wait()
+    Popen(['run_workers.py'])

requirements.txt

@@ -4,6 +4,7 @@ pypssl
 pypdns
 pyeupi
 dnspython
-bgpranking-web
-urlquery
 beautifulsoup4
+
+git+https://github.com/D4-project/IPASN-History.git/#egg=pyipasnhistory&subdirectory=client
+git+https://github.com/D4-project/BGP-Ranking.git/#egg=pybgpranking&subdirectory=client

setup.py

@@ -12,7 +12,7 @@ setup(
     url='https://github.com/CIRCL/url-abuse/',
     description='URL Abuse interface',
     packages=['urlabuse'],
-    scripts=['bin/run_backend.py', 'bin/run_workers.py'],
+    scripts=['bin/run_backend.py', 'bin/run_workers.py', 'bin/start.py'],
     classifiers=[
         'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
         'Development Status :: 3 - Alpha',

urlabuse/urlabuse.py

@@ -5,7 +5,7 @@
 # Copyright (C) 2014 CIRCL Computer Incident Response Center Luxembourg (SMILE gie)
 #
 
-from datetime import date
+from datetime import date, timedelta
 import json
 import redis
 from urllib.parse import quote
@@ -20,15 +20,10 @@ import re
 import sys
 import logging
 from pypdns import PyPDNS
-try:
+from pyipasnhistory import IPASNHistory
-    import bgpranking_web
+from pybgpranking import BGPRanking
-except Exception:
+
-    pass
 
-try:
-    import urlquery
-except Exception:
-    pass
 from pypssl import PyPSSL
 from pyeupi import PyEUPI
 import requests
@@ -344,6 +339,7 @@ def gsb_query(url, query):
         return response.text
 
 
+'''
 def urlquery_query(url, key, query):
     return None
     cached = _cache_get(query, 'urlquery')
@@ -366,6 +362,7 @@ def urlquery_query(url, key, query):
                 return total_alert_count
         else:
             return None
+'''
 
 
 def process_emails(emails, ignorelist, replacelist):
@@ -471,34 +468,36 @@ def eupi(url, key, q):
 
 
 def bgpranking(ip):
-    return None, None, None, None, None, None
+    cached = _cache_get(ip, 'ipasn')
-    cached = _cache_get(ip, 'bgp')
+    if cached is not None:
+        asn = cached['asn']
+        prefix = cached['prefix']
+    else:
+        ipasn = IPASNHistory()
+        response = ipasn.query(ip)
+        if 'response' not in response:
+            asn = None
+            prefix = None
+        entry = response['response'][list(response['response'].keys())[0]]
+        _cache_set(ip, entry, 'ipasn')
+        asn = entry['asn']
+        prefix = entry['prefix']
+
+    if not asn or not prefix:
+        # asn, prefix, asn_descr, rank, position, known_asns
+        return None, None, None, None, None, None
+
+    cached = _cache_get(asn, 'bgp')
     if cached is not None:
         return cached
-    details = bgpranking_web.ip_lookup(ip, 7)
+    bgpranking = BGPRanking()
-    ptrr = details.get('ptrrecord')
+    response = bgpranking.query(asn, date=(date.today() - timedelta(1)).isoformat())
-    if details.get('history') is None or len(details.get('history')) == 0:
+    if 'response' not in response:
-        return ptrr, None, None, None, None, None
+        return None, None, None, None, None, None
-    asn = details['history'][0].get('asn')
+    to_return = (asn, prefix, response['response']['asn_description'], response['response']['ranking']['rank'],
-    rank_info = bgpranking_web.cached_daily_rank(asn)
+                 response['response']['ranking']['position'], response['response']['ranking']['total_known_asns'])
-    position, total = bgpranking_web.cached_position(asn)
+    _cache_set(asn, to_return, 'bgp')
-    asn_descr = rank_info[1]
+    return to_return
-    rank = rank_info[-1]
-    if position:
-        position = int(position)
-    else:
-        position = -1
-    if total:
-        total = int(total)
-    else:
-        total = 0
-    if rank:
-        rank = float(rank)
-    else:
-        rank = -1
-    response = (ptrr, asn_descr, asn, position, total, rank)
-    _cache_set(ip, response, 'bgp')
-    return response
 
 
 def _deserialize_cached(entry):

website/3drparty.sh

@@ -5,7 +5,7 @@ set -x
 
 DEST_DIR="web/static/"
 
-ANGULAR='1.7.4'
+ANGULAR='1.7.5'
 ANGULAR_BOOTSTRAP='2.5.0'
 
 wget https://ajax.googleapis.com/ajax/libs/angularjs/${ANGULAR}/angular.min.js -O ${DEST_DIR}/angular.min.js

website/web/__init__.py

@@ -23,7 +23,7 @@ from urlabuse.helpers import get_socket_path
 import configparser
 from .proxied import ReverseProxied
 from urlabuse.urlabuse import is_valid_url, url_list, dns_resolve, phish_query, psslcircl, \
-    vt_query_url, gsb_query, urlquery_query, sphinxsearch, whois, pdnscircl, bgpranking, \
+    vt_query_url, gsb_query, sphinxsearch, whois, pdnscircl, bgpranking, \
     cached, get_mail_sent, set_mail_sent, get_submissions, eupi
 
 
@@ -224,6 +224,7 @@ def create_app(configfile=None):
         u = q.enqueue_call(func=gsb_query, args=(url, query,), result_ttl=500)
         return u.get_id()
 
+    '''
     @app.route('/urlquery', methods=['POST'])
     def urlquery():
         auth = read_auth('urlquery')
@@ -235,6 +236,7 @@ def create_app(configfile=None):
         query = data["query"]
         u = q.enqueue_call(func=urlquery_query, args=(url, key, query,), result_ttl=500)
         return u.get_id()
+    '''
 
     @app.route('/ticket', methods=['POST'])
     def ticket():

website/web/static/main.js

@@ -321,12 +321,12 @@
             link: function(scope, element, attrs) {
                 var get_response = function(jobID) {
                   globFct.poller(jobID, function(data){
-                      scope.ptr = data[0];
+                      scope.asndesc = data[2];
-                      scope.asndesc = data[1];
+                      scope.asn = data[0];
-                      scope.asn = data[2];
+                      scope.prefix = data[1];
-                      scope.position = data[3];
+                      scope.position = data[4];
-                      scope.total = data[4];
+                      scope.total = data[5];
-                      scope.value = data[5];
+                      scope.value = data[3];
                       if (scope.position < 100){
                           scope.alert_val = "danger";
                       } else if (scope.position < 1000){
@@ -338,7 +338,7 @@
                 };
                 globFct.query('bgpranking', {"query": scope.query}, get_response);
             },
-            template: '<div ng-show="asn" class="animate-show"><alert type="{{alert_val}}">Information from BGP Ranking: <ul><li ng-show="ptr">PTR Resource Record: {{ptr}}</li><li>Announced by: {{asndesc}} (<a target="_blank" ng-href="http://bgpranking.circl.lu/asn_details?asn={{asn}}">{{asn}}</a>)</li><li>This ASN is at position {{position}} in the list of {{total}} known ASNs ({{value}}).</li></ul></alert></div>'
+            template: '<div ng-show="asn" class="animate-show"><alert type="{{alert_val}}">Information from BGP Ranking: <ul><li>Announced by: {{asndesc}} (<a target="_blank" ng-href="http://bgpranking-ng.circl.lu/asn?asn={{asn}}">{{asn}}</a>)</li><li>This ASN is at position {{position}} in the list of {{total}} known ASNs ({{value}}).</li></ul></alert></div>'
         };
     });
 }());