new: Use new BGP Ranking & IPASN History

pull/14/head
Raphaël Vinot 2019-01-08 15:12:43 +01:00
parent 78871a7803
commit 8630ba8a98
7 changed files with 68 additions and 47 deletions

19
bin/start.py Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from subprocess import Popen
from urlabuse.helpers import get_homedir
import redis
import sys
if redis.VERSION < (3, ):
print('redis-py >= 3 is required.')
sys.exit()
if __name__ == '__main__':
# Just fail if the env isn't set.
get_homedir()
p = Popen(['run_backend.py', '--start'])
p.wait()
Popen(['run_workers.py'])

View File

@ -4,6 +4,7 @@ pypssl
pypdns pypdns
pyeupi pyeupi
dnspython dnspython
bgpranking-web
urlquery
beautifulsoup4 beautifulsoup4
git+https://github.com/D4-project/IPASN-History.git/#egg=pyipasnhistory&subdirectory=client
git+https://github.com/D4-project/BGP-Ranking.git/#egg=pybgpranking&subdirectory=client

View File

@ -12,7 +12,7 @@ setup(
url='https://github.com/CIRCL/url-abuse/', url='https://github.com/CIRCL/url-abuse/',
description='URL Abuse interface', description='URL Abuse interface',
packages=['urlabuse'], packages=['urlabuse'],
scripts=['bin/run_backend.py', 'bin/run_workers.py'], scripts=['bin/run_backend.py', 'bin/run_workers.py', 'bin/start.py'],
classifiers=[ classifiers=[
'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
'Development Status :: 3 - Alpha', 'Development Status :: 3 - Alpha',

View File

@ -5,7 +5,7 @@
# Copyright (C) 2014 CIRCL Computer Incident Response Center Luxembourg (SMILE gie) # Copyright (C) 2014 CIRCL Computer Incident Response Center Luxembourg (SMILE gie)
# #
from datetime import date from datetime import date, timedelta
import json import json
import redis import redis
from urllib.parse import quote from urllib.parse import quote
@ -20,15 +20,10 @@ import re
import sys import sys
import logging import logging
from pypdns import PyPDNS from pypdns import PyPDNS
try: from pyipasnhistory import IPASNHistory
import bgpranking_web from pybgpranking import BGPRanking
except Exception:
pass
try:
import urlquery
except Exception:
pass
from pypssl import PyPSSL from pypssl import PyPSSL
from pyeupi import PyEUPI from pyeupi import PyEUPI
import requests import requests
@ -344,6 +339,7 @@ def gsb_query(url, query):
return response.text return response.text
'''
def urlquery_query(url, key, query): def urlquery_query(url, key, query):
return None return None
cached = _cache_get(query, 'urlquery') cached = _cache_get(query, 'urlquery')
@ -366,6 +362,7 @@ def urlquery_query(url, key, query):
return total_alert_count return total_alert_count
else: else:
return None return None
'''
def process_emails(emails, ignorelist, replacelist): def process_emails(emails, ignorelist, replacelist):
@ -471,34 +468,36 @@ def eupi(url, key, q):
def bgpranking(ip): def bgpranking(ip):
return None, None, None, None, None, None cached = _cache_get(ip, 'ipasn')
cached = _cache_get(ip, 'bgp') if cached is not None:
asn = cached['asn']
prefix = cached['prefix']
else:
ipasn = IPASNHistory()
response = ipasn.query(ip)
if 'response' not in response:
asn = None
prefix = None
entry = response['response'][list(response['response'].keys())[0]]
_cache_set(ip, entry, 'ipasn')
asn = entry['asn']
prefix = entry['prefix']
if not asn or not prefix:
# asn, prefix, asn_descr, rank, position, known_asns
return None, None, None, None, None, None
cached = _cache_get(asn, 'bgp')
if cached is not None: if cached is not None:
return cached return cached
details = bgpranking_web.ip_lookup(ip, 7) bgpranking = BGPRanking()
ptrr = details.get('ptrrecord') response = bgpranking.query(asn, date=(date.today() - timedelta(1)).isoformat())
if details.get('history') is None or len(details.get('history')) == 0: if 'response' not in response:
return ptrr, None, None, None, None, None return None, None, None, None, None, None
asn = details['history'][0].get('asn') to_return = (asn, prefix, response['response']['asn_description'], response['response']['ranking']['rank'],
rank_info = bgpranking_web.cached_daily_rank(asn) response['response']['ranking']['position'], response['response']['ranking']['total_known_asns'])
position, total = bgpranking_web.cached_position(asn) _cache_set(asn, to_return, 'bgp')
asn_descr = rank_info[1] return to_return
rank = rank_info[-1]
if position:
position = int(position)
else:
position = -1
if total:
total = int(total)
else:
total = 0
if rank:
rank = float(rank)
else:
rank = -1
response = (ptrr, asn_descr, asn, position, total, rank)
_cache_set(ip, response, 'bgp')
return response
def _deserialize_cached(entry): def _deserialize_cached(entry):

View File

@ -5,7 +5,7 @@ set -x
DEST_DIR="web/static/" DEST_DIR="web/static/"
ANGULAR='1.7.4' ANGULAR='1.7.5'
ANGULAR_BOOTSTRAP='2.5.0' ANGULAR_BOOTSTRAP='2.5.0'
wget https://ajax.googleapis.com/ajax/libs/angularjs/${ANGULAR}/angular.min.js -O ${DEST_DIR}/angular.min.js wget https://ajax.googleapis.com/ajax/libs/angularjs/${ANGULAR}/angular.min.js -O ${DEST_DIR}/angular.min.js

View File

@ -23,7 +23,7 @@ from urlabuse.helpers import get_socket_path
import configparser import configparser
from .proxied import ReverseProxied from .proxied import ReverseProxied
from urlabuse.urlabuse import is_valid_url, url_list, dns_resolve, phish_query, psslcircl, \ from urlabuse.urlabuse import is_valid_url, url_list, dns_resolve, phish_query, psslcircl, \
vt_query_url, gsb_query, urlquery_query, sphinxsearch, whois, pdnscircl, bgpranking, \ vt_query_url, gsb_query, sphinxsearch, whois, pdnscircl, bgpranking, \
cached, get_mail_sent, set_mail_sent, get_submissions, eupi cached, get_mail_sent, set_mail_sent, get_submissions, eupi
@ -224,6 +224,7 @@ def create_app(configfile=None):
u = q.enqueue_call(func=gsb_query, args=(url, query,), result_ttl=500) u = q.enqueue_call(func=gsb_query, args=(url, query,), result_ttl=500)
return u.get_id() return u.get_id()
'''
@app.route('/urlquery', methods=['POST']) @app.route('/urlquery', methods=['POST'])
def urlquery(): def urlquery():
auth = read_auth('urlquery') auth = read_auth('urlquery')
@ -235,6 +236,7 @@ def create_app(configfile=None):
query = data["query"] query = data["query"]
u = q.enqueue_call(func=urlquery_query, args=(url, key, query,), result_ttl=500) u = q.enqueue_call(func=urlquery_query, args=(url, key, query,), result_ttl=500)
return u.get_id() return u.get_id()
'''
@app.route('/ticket', methods=['POST']) @app.route('/ticket', methods=['POST'])
def ticket(): def ticket():

View File

@ -321,12 +321,12 @@
link: function(scope, element, attrs) { link: function(scope, element, attrs) {
var get_response = function(jobID) { var get_response = function(jobID) {
globFct.poller(jobID, function(data){ globFct.poller(jobID, function(data){
scope.ptr = data[0]; scope.asndesc = data[2];
scope.asndesc = data[1]; scope.asn = data[0];
scope.asn = data[2]; scope.prefix = data[1];
scope.position = data[3]; scope.position = data[4];
scope.total = data[4]; scope.total = data[5];
scope.value = data[5]; scope.value = data[3];
if (scope.position < 100){ if (scope.position < 100){
scope.alert_val = "danger"; scope.alert_val = "danger";
} else if (scope.position < 1000){ } else if (scope.position < 1000){
@ -338,7 +338,7 @@
}; };
globFct.query('bgpranking', {"query": scope.query}, get_response); globFct.query('bgpranking', {"query": scope.query}, get_response);
}, },
template: '<div ng-show="asn" class="animate-show"><alert type="{{alert_val}}">Information from BGP Ranking: <ul><li ng-show="ptr">PTR Resource Record: {{ptr}}</li><li>Announced by: {{asndesc}} (<a target="_blank" ng-href="http://bgpranking.circl.lu/asn_details?asn={{asn}}">{{asn}}</a>)</li><li>This ASN is at position {{position}} in the list of {{total}} known ASNs ({{value}}).</li></ul></alert></div>' template: '<div ng-show="asn" class="animate-show"><alert type="{{alert_val}}">Information from BGP Ranking: <ul><li>Announced by: {{asndesc}} (<a target="_blank" ng-href="http://bgpranking-ng.circl.lu/asn?asn={{asn}}">{{asn}}</a>)</li><li>This ASN is at position {{position}} in the list of {{total}} known ASNs ({{value}}).</li></ul></alert></div>'
}; };
}); });
}()); }());