From 8630ba8a988114e3ec26624125de25c949093160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 8 Jan 2019 15:12:43 +0100 Subject: [PATCH] new: Use new BGP Ranking & IPASN History --- bin/start.py | 19 +++++++++++ requirements.txt | 5 +-- setup.py | 2 +- urlabuse/urlabuse.py | 69 +++++++++++++++++++------------------- website/3drparty.sh | 2 +- website/web/__init__.py | 4 ++- website/web/static/main.js | 14 ++++---- 7 files changed, 68 insertions(+), 47 deletions(-) create mode 100755 bin/start.py diff --git a/bin/start.py b/bin/start.py new file mode 100755 index 0000000..8286a30 --- /dev/null +++ b/bin/start.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from subprocess import Popen +from urlabuse.helpers import get_homedir + +import redis +import sys + +if redis.VERSION < (3, ): + print('redis-py >= 3 is required.') + sys.exit() + +if __name__ == '__main__': + # Just fail if the env isn't set. + get_homedir() + p = Popen(['run_backend.py', '--start']) + p.wait() + Popen(['run_workers.py']) diff --git a/requirements.txt b/requirements.txt index 3c08cde..e3bcf14 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ pypssl pypdns pyeupi dnspython -bgpranking-web -urlquery beautifulsoup4 + +git+https://github.com/D4-project/IPASN-History.git/#egg=pyipasnhistory&subdirectory=client +git+https://github.com/D4-project/BGP-Ranking.git/#egg=pybgpranking&subdirectory=client diff --git a/setup.py b/setup.py index 57e26b4..0255b34 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( url='https://github.com/CIRCL/url-abuse/', description='URL Abuse interface', packages=['urlabuse'], - scripts=['bin/run_backend.py', 'bin/run_workers.py'], + scripts=['bin/run_backend.py', 'bin/run_workers.py', 'bin/start.py'], classifiers=[ 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', 'Development Status :: 3 - Alpha', diff --git a/urlabuse/urlabuse.py b/urlabuse/urlabuse.py index 3871189..6d69fa7 100644 --- a/urlabuse/urlabuse.py +++ b/urlabuse/urlabuse.py @@ -5,7 +5,7 @@ # Copyright (C) 2014 CIRCL Computer Incident Response Center Luxembourg (SMILE gie) # -from datetime import date +from datetime import date, timedelta import json import redis from urllib.parse import quote @@ -20,15 +20,10 @@ import re import sys import logging from pypdns import PyPDNS -try: - import bgpranking_web -except Exception: - pass +from pyipasnhistory import IPASNHistory +from pybgpranking import BGPRanking + -try: - import urlquery -except Exception: - pass from pypssl import PyPSSL from pyeupi import PyEUPI import requests @@ -344,6 +339,7 @@ def gsb_query(url, query): return response.text +''' def urlquery_query(url, key, query): return None cached = _cache_get(query, 'urlquery') @@ -366,6 +362,7 @@ def urlquery_query(url, key, query): return total_alert_count else: return None +''' def process_emails(emails, ignorelist, replacelist): @@ -471,34 +468,36 @@ def eupi(url, key, q): def bgpranking(ip): - return None, None, None, None, None, None - cached = _cache_get(ip, 'bgp') + cached = _cache_get(ip, 'ipasn') + if cached is not None: + asn = cached['asn'] + prefix = cached['prefix'] + else: + ipasn = IPASNHistory() + response = ipasn.query(ip) + if 'response' not in response: + asn = None + prefix = None + entry = response['response'][list(response['response'].keys())[0]] + _cache_set(ip, entry, 'ipasn') + asn = entry['asn'] + prefix = entry['prefix'] + + if not asn or not prefix: + # asn, prefix, asn_descr, rank, position, known_asns + return None, None, None, None, None, None + + cached = _cache_get(asn, 'bgp') if cached is not None: return cached - details = bgpranking_web.ip_lookup(ip, 7) - ptrr = details.get('ptrrecord') - if details.get('history') is None or len(details.get('history')) == 0: - return ptrr, None, None, None, None, None - asn = details['history'][0].get('asn') - rank_info = bgpranking_web.cached_daily_rank(asn) - position, total = bgpranking_web.cached_position(asn) - asn_descr = rank_info[1] - rank = rank_info[-1] - if position: - position = int(position) - else: - position = -1 - if total: - total = int(total) - else: - total = 0 - if rank: - rank = float(rank) - else: - rank = -1 - response = (ptrr, asn_descr, asn, position, total, rank) - _cache_set(ip, response, 'bgp') - return response + bgpranking = BGPRanking() + response = bgpranking.query(asn, date=(date.today() - timedelta(1)).isoformat()) + if 'response' not in response: + return None, None, None, None, None, None + to_return = (asn, prefix, response['response']['asn_description'], response['response']['ranking']['rank'], + response['response']['ranking']['position'], response['response']['ranking']['total_known_asns']) + _cache_set(asn, to_return, 'bgp') + return to_return def _deserialize_cached(entry): diff --git a/website/3drparty.sh b/website/3drparty.sh index b873bc3..e345a98 100755 --- a/website/3drparty.sh +++ b/website/3drparty.sh @@ -5,7 +5,7 @@ set -x DEST_DIR="web/static/" -ANGULAR='1.7.4' +ANGULAR='1.7.5' ANGULAR_BOOTSTRAP='2.5.0' wget https://ajax.googleapis.com/ajax/libs/angularjs/${ANGULAR}/angular.min.js -O ${DEST_DIR}/angular.min.js diff --git a/website/web/__init__.py b/website/web/__init__.py index eb99aa6..a28d824 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -23,7 +23,7 @@ from urlabuse.helpers import get_socket_path import configparser from .proxied import ReverseProxied from urlabuse.urlabuse import is_valid_url, url_list, dns_resolve, phish_query, psslcircl, \ - vt_query_url, gsb_query, urlquery_query, sphinxsearch, whois, pdnscircl, bgpranking, \ + vt_query_url, gsb_query, sphinxsearch, whois, pdnscircl, bgpranking, \ cached, get_mail_sent, set_mail_sent, get_submissions, eupi @@ -224,6 +224,7 @@ def create_app(configfile=None): u = q.enqueue_call(func=gsb_query, args=(url, query,), result_ttl=500) return u.get_id() + ''' @app.route('/urlquery', methods=['POST']) def urlquery(): auth = read_auth('urlquery') @@ -235,6 +236,7 @@ def create_app(configfile=None): query = data["query"] u = q.enqueue_call(func=urlquery_query, args=(url, key, query,), result_ttl=500) return u.get_id() + ''' @app.route('/ticket', methods=['POST']) def ticket(): diff --git a/website/web/static/main.js b/website/web/static/main.js index 29324c8..300fd6e 100644 --- a/website/web/static/main.js +++ b/website/web/static/main.js @@ -321,12 +321,12 @@ link: function(scope, element, attrs) { var get_response = function(jobID) { globFct.poller(jobID, function(data){ - scope.ptr = data[0]; - scope.asndesc = data[1]; - scope.asn = data[2]; - scope.position = data[3]; - scope.total = data[4]; - scope.value = data[5]; + scope.asndesc = data[2]; + scope.asn = data[0]; + scope.prefix = data[1]; + scope.position = data[4]; + scope.total = data[5]; + scope.value = data[3]; if (scope.position < 100){ scope.alert_val = "danger"; } else if (scope.position < 1000){ @@ -338,7 +338,7 @@ }; globFct.query('bgpranking', {"query": scope.query}, get_response); }, - template: '
Information from BGP Ranking:
  • PTR Resource Record: {{ptr}}
  • Announced by: {{asndesc}} ({{asn}})
  • This ASN is at position {{position}} in the list of {{total}} known ASNs ({{value}}).
' + template: '
Information from BGP Ranking:
  • Announced by: {{asndesc}} ({{asn}})
  • This ASN is at position {{position}} in the list of {{total}} known ASNs ({{value}}).
' }; }); }());