CIRCL
/
url-abuse
mirror of https://github.com/CIRCL/url-abuse
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Sync with master

pull/14/head
Raphaël Vinot 2018-11-16 10:26:55 +01:00
parent 62c1328075
commit 150a908aac
5 changed files with 89 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
update_deps.sh
View File

@ -5,13 +5,13 @@ set -x
DEST_DIR="web/static/" DEST_DIR="web/static/"
ANGULAR='1.4.5' ANGULAR='1.7.4'
ANGULAR_BOOTSTRAP='0.13.4' ANGULAR_BOOTSTRAP='2.5.0'
wget https://ajax.googleapis.com/ajax/libs/angularjs/${ANGULAR}/angular.min.js -O ${DEST_DIR}/angular.min.js wget https://ajax.googleapis.com/ajax/libs/angularjs/${ANGULAR}/angular.min.js -O ${DEST_DIR}/angular.min.js
wget https://angular-ui.github.io/bootstrap/ui-bootstrap-tpls-${ANGULAR_BOOTSTRAP}.min.js -O ${DEST_DIR}/ui-bootstrap-tpls.min.js wget https://angular-ui.github.io/bootstrap/ui-bootstrap-tpls-${ANGULAR_BOOTSTRAP}.min.js -O ${DEST_DIR}/ui-bootstrap-tpls.min.js
wget https://sphinxsearch.googlecode.com/svn/trunk/api/sphinxapi.py -O sphinxapi.py wget https://raw.githubusercontent.com/sphinxsearch/sphinx/master/api/sphinxapi.py -O sphinxapi.py

53
url_abuse_async.py
View File

@ -20,15 +20,15 @@ import re
import sys import sys
import logging import logging
from pypdns import PyPDNS from pypdns import PyPDNS
import bgpranking_web # import bgpranking_web
import urlquery # import urlquery
from pypssl import PyPSSL from pypssl import PyPSSL
from pyeupi import PyEUPI from pyeupi import PyEUPI
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
try: try:
import sphinxapi # import sphinxapi
sphinx = True sphinx = True
except: except:
sphinx = False sphinx = False
@ -40,7 +40,7 @@ r_cache = None
def _cache_init(host='localhost', port=6334, db=1): def _cache_init(host='localhost', port=6334, db=1):
global r_cache global r_cache
if enable_cache and r_cache is None: if enable_cache and r_cache is None:
r_cache = redis.Redis(host, port, db=db) r_cache = redis.Redis(host, port, db=db, decode_responses=True)
def _cache_set(key, value, field=None): def _cache_set(key, value, field=None):
@ -128,7 +128,12 @@ def is_valid_url(url):
def is_ip(host): def is_ip(host):
if ':' in host: if isinstance(host, bytes):
to_search = b':'
else:
to_search = ':'
if to_search in host:
try: try:
socket.inet_pton(socket.AF_INET6, host) socket.inet_pton(socket.AF_INET6, host)
return True return True
@ -173,8 +178,11 @@ def get_urls(url, depth=1):
out = result["content"].split(";") out = result["content"].split(";")
if len(out) == 2: if len(out) == 2:
wait, text = out wait, text = out
a, url = text.split('=', 1) try:
return url.strip() a, url = text.split('=', 1)
return url.strip()
except:
print(text)
return None return None
resolve, reason = try_resolve(fex, url) resolve, reason = try_resolve(fex, url)
@ -240,7 +248,7 @@ def dns_resolve(url):
return cached return cached
fex = Faup() fex = Faup()
fex.decode(url) fex.decode(url)
host = fex.get_host().lower() host = fex.get_host().decode().lower()
ipv4 = None ipv4 = None
ipv6 = None ipv6 = None
if is_ip(host): if is_ip(host):
@ -291,6 +299,9 @@ def phish_query(url, key, query):
def sphinxsearch(server, port, url, query): def sphinxsearch(server, port, url, query):
# WARNING: too dangerous to have on the public interface
return ''
"""
if not sphinx: if not sphinx:
return None return None
cached = _cache_get(query, 'sphinx') cached = _cache_get(query, 'sphinx')
@ -310,6 +321,8 @@ def sphinxsearch(server, port, url, query):
_cache_set(query, result, 'sphinx') _cache_set(query, result, 'sphinx')
return result return result
"""
def vt_query_url(url, url_up, key, query, upload=True): def vt_query_url(url, url_up, key, query, upload=True):
cached = _cache_get(query, 'vt') cached = _cache_get(query, 'vt')
@ -344,6 +357,7 @@ def gsb_query(url, query):
def urlquery_query(url, key, query): def urlquery_query(url, key, query):
return None
cached = _cache_get(query, 'urlquery') cached = _cache_get(query, 'urlquery')
if cached is not None: if cached is not None:
return cached return cached
@ -400,15 +414,15 @@ def whois(server, port, domain, ignorelist, replacelist):
d = fex.get_domain().lower() d = fex.get_domain().lower()
else: else:
d = domain d = domain
s.send(d + "\r\n") s.send(("{}\r\n".format(d)).encode())
response = '' response = b''
while True: while True:
d = s.recv(4096) d = s.recv(4096)
response += d response += d
if d == '': if d == b'':
break break
s.close() s.close()
match = re.findall(r'[\w\.-]+@[\w\.-]+', response) match = re.findall(r'[\w\.-]+@[\w\.-]+', response.decode())
emails = process_emails(match, ignorelist, replacelist) emails = process_emails(match, ignorelist, replacelist)
if len(emails) == 0: if len(emails) == 0:
return None return None
@ -469,6 +483,7 @@ def eupi(url, key, q):
def bgpranking(ip): def bgpranking(ip):
return None, None, None, None, None, None
cached = _cache_get(ip, 'bgp') cached = _cache_get(ip, 'bgp')
if cached is not None: if cached is not None:
return cached return cached
@ -481,7 +496,19 @@ def bgpranking(ip):
position, total = bgpranking_web.cached_position(asn) position, total = bgpranking_web.cached_position(asn)
asn_descr = rank_info[1] asn_descr = rank_info[1]
rank = rank_info[-1] rank = rank_info[-1]
response = (ptrr, asn_descr, asn, int(position), int(total), float(rank)) if position:
position = int(position)
else:
position = -1
if total:
total = int(total)
else:
total = 0
if rank:
rank = float(rank)
else:
rank = -1
response = (ptrr, asn_descr, asn, position, total, rank)
_cache_set(ip, response, 'bgp') _cache_set(ip, response, 'bgp')
return response return response

36
web/__init__.py
View File

@ -153,32 +153,32 @@ def create_app(configfile=None):
@app.route('/start', methods=['POST']) @app.route('/start', methods=['POST'])
def run_query(): def run_query():
data = json.loads(request.data) data = json.loads(request.data.decode())
url = data["url"] url = data["url"]
ip = _get_user_ip(request) ip = _get_user_ip(request)
app.logger.info('{} {}'.format(ip, url)) app.logger.info('{} {}'.format(ip, url))
if get_submissions(url) >= autosend_threshold: if get_submissions(url) and get_submissions(url) >= autosend_threshold:
send(url, '', True) send(url, '', True)
is_valid = q.enqueue_call(func=is_valid_url, args=(url,), result_ttl=500) is_valid = q.enqueue_call(func=is_valid_url, args=(url,), result_ttl=500)
return is_valid.get_id() return is_valid.get_id()
@app.route('/urls', methods=['POST']) @app.route('/urls', methods=['POST'])
def urls(): def urls():
data = json.loads(request.data) data = json.loads(request.data.decode())
url = data["url"] url = data["url"]
u = q.enqueue_call(func=url_list, args=(url,), result_ttl=500) u = q.enqueue_call(func=url_list, args=(url,), result_ttl=500)
return u.get_id() return u.get_id()
@app.route('/resolve', methods=['POST']) @app.route('/resolve', methods=['POST'])
def resolve(): def resolve():
data = json.loads(request.data) data = json.loads(request.data.decode())
url = data["url"] url = data["url"]
u = q.enqueue_call(func=dns_resolve, args=(url,), result_ttl=500) u = q.enqueue_call(func=dns_resolve, args=(url,), result_ttl=500)
return u.get_id() return u.get_id()
@app.route('/phishtank', methods=['POST']) @app.route('/phishtank', methods=['POST'])
def phishtank(): def phishtank():
data = json.loads(request.data) data = json.loads(request.data.decode())
if not os.path.exists('phishtank.key'): if not os.path.exists('phishtank.key'):
return None return None
url = parser.get("PHISHTANK", "url") url = parser.get("PHISHTANK", "url")
@ -189,7 +189,7 @@ def create_app(configfile=None):
@app.route('/virustotal_report', methods=['POST']) @app.route('/virustotal_report', methods=['POST'])
def vt(): def vt():
data = json.loads(request.data) data = json.loads(request.data.decode())
if not os.path.exists('virustotal.key'): if not os.path.exists('virustotal.key'):
return None return None
url = parser.get("VIRUSTOTAL", "url_report") url = parser.get("VIRUSTOTAL", "url_report")
@ -201,7 +201,7 @@ def create_app(configfile=None):
@app.route('/googlesafebrowsing', methods=['POST']) @app.route('/googlesafebrowsing', methods=['POST'])
def gsb(): def gsb():
data = json.loads(request.data) data = json.loads(request.data.decode())
if not os.path.exists('googlesafebrowsing.key'): if not os.path.exists('googlesafebrowsing.key'):
return None return None
url = parser.get("GOOGLESAFEBROWSING", "url") url = parser.get("GOOGLESAFEBROWSING", "url")
@ -213,7 +213,7 @@ def create_app(configfile=None):
@app.route('/urlquery', methods=['POST']) @app.route('/urlquery', methods=['POST'])
def urlquery(): def urlquery():
data = json.loads(request.data) data = json.loads(request.data.decode())
if not os.path.exists('urlquery.key'): if not os.path.exists('urlquery.key'):
return None return None
url = parser.get("URLQUERY", "url") url = parser.get("URLQUERY", "url")
@ -226,7 +226,7 @@ def create_app(configfile=None):
def ticket(): def ticket():
if not request.authorization: if not request.authorization:
return '' return ''
data = json.loads(request.data) data = json.loads(request.data.decode())
server = parser.get("SPHINX", "server") server = parser.get("SPHINX", "server")
port = int(parser.get("SPHINX", "port")) port = int(parser.get("SPHINX", "port"))
url = parser.get("ITS", "url") url = parser.get("ITS", "url")
@ -237,11 +237,11 @@ def create_app(configfile=None):
@app.route('/whois', methods=['POST']) @app.route('/whois', methods=['POST'])
def whoismail(): def whoismail():
if not request.authorization: # if not request.authorization:
return '' # return ''
server = parser.get("WHOIS", "server") server = parser.get("WHOIS", "server")
port = parser.getint("WHOIS", "port") port = parser.getint("WHOIS", "port")
data = json.loads(request.data) data = json.loads(request.data.decode())
query = data["query"] query = data["query"]
u = q.enqueue_call(func=whois, args=(server, port, query, ignorelist, replacelist), u = q.enqueue_call(func=whois, args=(server, port, query, ignorelist, replacelist),
result_ttl=500) result_ttl=500)
@ -249,7 +249,7 @@ def create_app(configfile=None):
@app.route('/eupi', methods=['POST']) @app.route('/eupi', methods=['POST'])
def eu(): def eu():
data = json.loads(request.data) data = json.loads(request.data.decode())
if not os.path.exists('eupi.key'): if not os.path.exists('eupi.key'):
return None return None
url = parser.get("EUPI", "url") url = parser.get("EUPI", "url")
@ -262,7 +262,7 @@ def create_app(configfile=None):
def dnscircl(): def dnscircl():
url = parser.get("PDNS_CIRCL", "url") url = parser.get("PDNS_CIRCL", "url")
user, password = open('pdnscircl.key', 'r').readlines() user, password = open('pdnscircl.key', 'r').readlines()
data = json.loads(request.data) data = json.loads(request.data.decode())
query = data["query"] query = data["query"]
u = q.enqueue_call(func=pdnscircl, args=(url, user.strip(), password.strip(), u = q.enqueue_call(func=pdnscircl, args=(url, user.strip(), password.strip(),
query,), result_ttl=500) query,), result_ttl=500)
@ -270,7 +270,7 @@ def create_app(configfile=None):
@app.route('/bgpranking', methods=['POST']) @app.route('/bgpranking', methods=['POST'])
def bgpr(): def bgpr():
data = json.loads(request.data) data = json.loads(request.data.decode())
query = data["query"] query = data["query"]
u = q.enqueue_call(func=bgpranking, args=(query,), result_ttl=500) u = q.enqueue_call(func=bgpranking, args=(query,), result_ttl=500)
return u.get_id() return u.get_id()
@ -279,7 +279,7 @@ def create_app(configfile=None):
def sslcircl(): def sslcircl():
url = parser.get("PSSL_CIRCL", "url") url = parser.get("PSSL_CIRCL", "url")
user, password = open('psslcircl.key', 'r').readlines() user, password = open('psslcircl.key', 'r').readlines()
data = json.loads(request.data) data = json.loads(request.data.decode())
query = data["query"] query = data["query"]
u = q.enqueue_call(func=psslcircl, args=(url, user.strip(), password.strip(), u = q.enqueue_call(func=psslcircl, args=(url, user.strip(), password.strip(),
query,), result_ttl=500) query,), result_ttl=500)
@ -287,7 +287,7 @@ def create_app(configfile=None):
@app.route('/get_cache', methods=['POST']) @app.route('/get_cache', methods=['POST'])
def get_cache(): def get_cache():
data = json.loads(request.data) data = json.loads(request.data.decode())
url = data["query"] url = data["query"]
data = cached(url) data = cached(url)
dumped = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')) dumped = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))
@ -347,7 +347,7 @@ def create_app(configfile=None):
@app.route('/submit', methods=['POST']) @app.route('/submit', methods=['POST'])
def send_mail(): def send_mail():
data = json.loads(request.data) data = json.loads(request.data.decode())
url = data["url"] url = data["url"]
if not get_mail_sent(url): if not get_mail_sent(url):
ip = _get_user_ip(request) ip = _get_user_ip(request)

22
web/static/main.js
View File

@ -26,15 +26,20 @@
poller: function myself(jobID, callback) { poller: function myself(jobID, callback) {
var timeout = ""; var timeout = "";
// fire another request // fire another request
$http.get('_result/' + jobID). $http.get('_result/' + jobID.data).
success(function(data, status, headers, config) { then(function(data) {
if(status === 202) { if(data.status === 202) {
$log.log(data, status); $log.log(data, status);
} else if (status === 200){ } else if (data.status === 200){
$log.log(data); $log.log(data.data);
$timeout.cancel(timeout); $timeout.cancel(timeout);
callback(angular.fromJson(data)); if (data.data === "null"){
return; $log.log('Got null data');
return;
} else {
callback(data.data);
return;
};
} }
// continue to call the poller() function every 2 seconds // continue to call the poller() function every 2 seconds
// until the timout is cancelled // until the timout is cancelled
@ -43,8 +48,7 @@
}, },
query: function(path, data, callback) { query: function(path, data, callback) {
$http.post(path, data). $http.post(path, data).
success(callback). then(callback, function(error) {
error(function(error) {
$log.log(error); $log.log(error);
}); });
} }

30
web/templates/url-report.html
View File

@ -1,8 +1,8 @@
{% raw %} {% raw %}
<accordion> <uib-accordion>
<accordion-group is-open=true> <div uib-accordion-group is-open=true>
<accordion-heading>{{url}}</accordion-heading> <uib-accordion-heading>{{url}}</uib-accordion-heading>
<uq-phishtank data="url"></uq-phishtank> <uq-phishtank data="url"></uq-phishtank>
<uq-virustotal data="url"></uq-virustotal> <uq-virustotal data="url"></uq-virustotal>
<uq-googlesafebrowsing data="url"></uq-googlesafebrowsing> <uq-googlesafebrowsing data="url"></uq-googlesafebrowsing>
@ -12,9 +12,9 @@
<uq-whois data="url"></uq-whois><br> <uq-whois data="url"></uq-whois><br>
<div ng-repeat="ip in ipv4"> <div ng-repeat="ip in ipv4">
<accordion> <uib-accordion>
<accordion-group is-open=true> <div uib-accordion-group is-open=true>
<accordion-heading>{{ip}}</accordion-heading> <uib-accordion-heading>{{ip}}</uib-accordion-heading>
<uq-phishtank data="ip"></uq-phishtank> <uq-phishtank data="ip"></uq-phishtank>
<!-- <li><uq-virustotal data="ip"></uq-virustotal></li> --> <!-- <li><uq-virustotal data="ip"></uq-virustotal></li> -->
<uq-bgpranking data="ip"></uq-bgpranking> <uq-bgpranking data="ip"></uq-bgpranking>
@ -23,14 +23,14 @@
<uq-psslcircl data="ip"></uq-psslcircl> <uq-psslcircl data="ip"></uq-psslcircl>
<uq-ticket data="ip"></uq-ticket> <uq-ticket data="ip"></uq-ticket>
<uq-whois data="ip"></uq-whois> <uq-whois data="ip"></uq-whois>
</accordion-group> </div>
</accordion> </uib-accordion>
</div> </div>
<div ng-repeat="ip in ipv6"> <div ng-repeat="ip in ipv6">
<accordion> <uib-accordion>
<accordion-group is-open=true> <div uib-accordion-group is-open=true>
<accordion-heading>{{ip}}</accordion-heading> <uib-accordion-heading>{{ip}}</uib-accordion-heading>
<uq-phishtank data="ip"></uq-phishtank> <uq-phishtank data="ip"></uq-phishtank>
<!-- <li><uq-virustotal data="ip"></uq-virustotal></li> --> <!-- <li><uq-virustotal data="ip"></uq-virustotal></li> -->
<!--<li><uq-bgpranking data="ip"></uq-bgpranking></li>--> <!--<li><uq-bgpranking data="ip"></uq-bgpranking></li>-->
@ -38,10 +38,10 @@
<uq-pdnscircl data="ip"></uq-pdnscircl> <uq-pdnscircl data="ip"></uq-pdnscircl>
<uq-ticket data="ip"></uq-ticket> <uq-ticket data="ip"></uq-ticket>
<uq-whois data="ip"></uq-whois> <uq-whois data="ip"></uq-whois>
</accordion-group> </div>
</accordion> </uib-accordion>
</div> </div>
</accordion-group> </div>
</accordion> </uib-accordion>
{% endraw %} {% endraw %}