CIRCL
/
url-abuse
mirror of https://github.com/CIRCL/url-abuse
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Sync with master

pull/14/head
Raphaël Vinot 2018-11-16 10:26:55 +01:00
parent 62c1328075
commit 150a908aac
5 changed files with 89 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
update_deps.sh
View File

@ -5,13 +5,13 @@ set -x
DEST_DIR="web/static/"
ANGULAR='1.4.5'
ANGULAR_BOOTSTRAP='0.13.4'
ANGULAR='1.7.4'
ANGULAR_BOOTSTRAP='2.5.0'
wget https://ajax.googleapis.com/ajax/libs/angularjs/${ANGULAR}/angular.min.js -O ${DEST_DIR}/angular.min.js
wget https://angular-ui.github.io/bootstrap/ui-bootstrap-tpls-${ANGULAR_BOOTSTRAP}.min.js -O ${DEST_DIR}/ui-bootstrap-tpls.min.js
wget https://sphinxsearch.googlecode.com/svn/trunk/api/sphinxapi.py -O sphinxapi.py
wget https://raw.githubusercontent.com/sphinxsearch/sphinx/master/api/sphinxapi.py -O sphinxapi.py

53
url_abuse_async.py
View File

@ -20,15 +20,15 @@ import re
import sys
import logging
from pypdns import PyPDNS
import bgpranking_web
import urlquery
# import bgpranking_web
# import urlquery
from pypssl import PyPSSL
from pyeupi import PyEUPI
import requests
from bs4 import BeautifulSoup
try:
import sphinxapi
# import sphinxapi
sphinx = True
except:
sphinx = False
@ -40,7 +40,7 @@ r_cache = None
def _cache_init(host='localhost', port=6334, db=1):
global r_cache
if enable_cache and r_cache is None:
r_cache = redis.Redis(host, port, db=db)
r_cache = redis.Redis(host, port, db=db, decode_responses=True)
def _cache_set(key, value, field=None):
@ -128,7 +128,12 @@ def is_valid_url(url):
def is_ip(host):
if ':' in host:
if isinstance(host, bytes):
to_search = b':'
else:
to_search = ':'
if to_search in host:
try:
socket.inet_pton(socket.AF_INET6, host)
return True
@ -173,8 +178,11 @@ def get_urls(url, depth=1):
out = result["content"].split(";")
if len(out) == 2:
wait, text = out
a, url = text.split('=', 1)
return url.strip()
try:
a, url = text.split('=', 1)
return url.strip()
except:
print(text)
return None
resolve, reason = try_resolve(fex, url)
@ -240,7 +248,7 @@ def dns_resolve(url):
return cached
fex = Faup()
fex.decode(url)
host = fex.get_host().lower()
host = fex.get_host().decode().lower()
ipv4 = None
ipv6 = None
if is_ip(host):
@ -291,6 +299,9 @@ def phish_query(url, key, query):
def sphinxsearch(server, port, url, query):
# WARNING: too dangerous to have on the public interface
return ''
"""
if not sphinx:
return None
cached = _cache_get(query, 'sphinx')
@ -310,6 +321,8 @@ def sphinxsearch(server, port, url, query):
_cache_set(query, result, 'sphinx')
return result
"""
def vt_query_url(url, url_up, key, query, upload=True):
cached = _cache_get(query, 'vt')
@ -344,6 +357,7 @@ def gsb_query(url, query):
def urlquery_query(url, key, query):
return None
cached = _cache_get(query, 'urlquery')
if cached is not None:
return cached
@ -400,15 +414,15 @@ def whois(server, port, domain, ignorelist, replacelist):
d = fex.get_domain().lower()
else:
d = domain
s.send(d + "\r\n")
response = ''
s.send(("{}\r\n".format(d)).encode())
response = b''
while True:
d = s.recv(4096)
response += d
if d == '':
if d == b'':
break
s.close()
match = re.findall(r'[\w\.-]+@[\w\.-]+', response)
match = re.findall(r'[\w\.-]+@[\w\.-]+', response.decode())
emails = process_emails(match, ignorelist, replacelist)
if len(emails) == 0:
return None
@ -469,6 +483,7 @@ def eupi(url, key, q):
def bgpranking(ip):
return None, None, None, None, None, None
cached = _cache_get(ip, 'bgp')
if cached is not None:
return cached
@ -481,7 +496,19 @@ def bgpranking(ip):
position, total = bgpranking_web.cached_position(asn)
asn_descr = rank_info[1]
rank = rank_info[-1]
response = (ptrr, asn_descr, asn, int(position), int(total), float(rank))
if position:
position = int(position)
else:
position = -1
if total:
total = int(total)
else:
total = 0
if rank:
rank = float(rank)
else:
rank = -1
response = (ptrr, asn_descr, asn, position, total, rank)
_cache_set(ip, response, 'bgp')
return response

36
web/__init__.py
View File

@ -153,32 +153,32 @@ def create_app(configfile=None):
@app.route('/start', methods=['POST'])
def run_query():
data = json.loads(request.data)
data = json.loads(request.data.decode())
url = data["url"]
ip = _get_user_ip(request)
app.logger.info('{} {}'.format(ip, url))
if get_submissions(url) >= autosend_threshold:
if get_submissions(url) and get_submissions(url) >= autosend_threshold:
send(url, '', True)
is_valid = q.enqueue_call(func=is_valid_url, args=(url,), result_ttl=500)
return is_valid.get_id()
@app.route('/urls', methods=['POST'])
def urls():
data = json.loads(request.data)
data = json.loads(request.data.decode())
url = data["url"]
u = q.enqueue_call(func=url_list, args=(url,), result_ttl=500)
return u.get_id()
@app.route('/resolve', methods=['POST'])
def resolve():
data = json.loads(request.data)
data = json.loads(request.data.decode())
url = data["url"]
u = q.enqueue_call(func=dns_resolve, args=(url,), result_ttl=500)
return u.get_id()
@app.route('/phishtank', methods=['POST'])
def phishtank():
data = json.loads(request.data)
data = json.loads(request.data.decode())
if not os.path.exists('phishtank.key'):
return None
url = parser.get("PHISHTANK", "url")
@ -189,7 +189,7 @@ def create_app(configfile=None):
@app.route('/virustotal_report', methods=['POST'])
def vt():
data = json.loads(request.data)
data = json.loads(request.data.decode())
if not os.path.exists('virustotal.key'):
return None
url = parser.get("VIRUSTOTAL", "url_report")
@ -201,7 +201,7 @@ def create_app(configfile=None):
@app.route('/googlesafebrowsing', methods=['POST'])
def gsb():
data = json.loads(request.data)
data = json.loads(request.data.decode())
if not os.path.exists('googlesafebrowsing.key'):
return None
url = parser.get("GOOGLESAFEBROWSING", "url")
@ -213,7 +213,7 @@ def create_app(configfile=None):
@app.route('/urlquery', methods=['POST'])
def urlquery():
data = json.loads(request.data)
data = json.loads(request.data.decode())
if not os.path.exists('urlquery.key'):
return None
url = parser.get("URLQUERY", "url")
@ -226,7 +226,7 @@ def create_app(configfile=None):
def ticket():
if not request.authorization:
return ''
data = json.loads(request.data)
data = json.loads(request.data.decode())
server = parser.get("SPHINX", "server")
port = int(parser.get("SPHINX", "port"))
url = parser.get("ITS", "url")
@ -237,11 +237,11 @@ def create_app(configfile=None):
@app.route('/whois', methods=['POST'])
def whoismail():
if not request.authorization:
return ''
# if not request.authorization:
# return ''
server = parser.get("WHOIS", "server")
port = parser.getint("WHOIS", "port")
data = json.loads(request.data)
data = json.loads(request.data.decode())
query = data["query"]
u = q.enqueue_call(func=whois, args=(server, port, query, ignorelist, replacelist),
result_ttl=500)
@ -249,7 +249,7 @@ def create_app(configfile=None):
@app.route('/eupi', methods=['POST'])
def eu():
data = json.loads(request.data)
data = json.loads(request.data.decode())
if not os.path.exists('eupi.key'):
return None
url = parser.get("EUPI", "url")
@ -262,7 +262,7 @@ def create_app(configfile=None):
def dnscircl():
url = parser.get("PDNS_CIRCL", "url")
user, password = open('pdnscircl.key', 'r').readlines()
data = json.loads(request.data)
data = json.loads(request.data.decode())
query = data["query"]
u = q.enqueue_call(func=pdnscircl, args=(url, user.strip(), password.strip(),
query,), result_ttl=500)
@ -270,7 +270,7 @@ def create_app(configfile=None):
@app.route('/bgpranking', methods=['POST'])
def bgpr():
data = json.loads(request.data)
data = json.loads(request.data.decode())
query = data["query"]
u = q.enqueue_call(func=bgpranking, args=(query,), result_ttl=500)
return u.get_id()
@ -279,7 +279,7 @@ def create_app(configfile=None):
def sslcircl():
url = parser.get("PSSL_CIRCL", "url")
user, password = open('psslcircl.key', 'r').readlines()
data = json.loads(request.data)
data = json.loads(request.data.decode())
query = data["query"]
u = q.enqueue_call(func=psslcircl, args=(url, user.strip(), password.strip(),
query,), result_ttl=500)
@ -287,7 +287,7 @@ def create_app(configfile=None):
@app.route('/get_cache', methods=['POST'])
def get_cache():
data = json.loads(request.data)
data = json.loads(request.data.decode())
url = data["query"]
data = cached(url)
dumped = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))
@ -347,7 +347,7 @@ def create_app(configfile=None):
@app.route('/submit', methods=['POST'])
def send_mail():
data = json.loads(request.data)
data = json.loads(request.data.decode())
url = data["url"]
if not get_mail_sent(url):
ip = _get_user_ip(request)

22
web/static/main.js
View File

@ -26,15 +26,20 @@
poller: function myself(jobID, callback) {
var timeout = "";
// fire another request
$http.get('_result/' + jobID).
success(function(data, status, headers, config) {
if(status === 202) {
$http.get('_result/' + jobID.data).
then(function(data) {
if(data.status === 202) {
$log.log(data, status);
} else if (status === 200){
$log.log(data);
} else if (data.status === 200){
$log.log(data.data);
$timeout.cancel(timeout);
callback(angular.fromJson(data));
return;
if (data.data === "null"){
$log.log('Got null data');
return;
} else {
callback(data.data);
return;
};
}
// continue to call the poller() function every 2 seconds
// until the timout is cancelled
@ -43,8 +48,7 @@
},
query: function(path, data, callback) {
$http.post(path, data).
success(callback).
error(function(error) {
then(callback, function(error) {
$log.log(error);
});
}

30
web/templates/url-report.html
View File

@ -1,8 +1,8 @@
{% raw %}
<accordion>
<accordion-group is-open=true>
<accordion-heading>{{url}}</accordion-heading>
<uib-accordion>
<div uib-accordion-group is-open=true>
<uib-accordion-heading>{{url}}</uib-accordion-heading>
<uq-phishtank data="url"></uq-phishtank>
<uq-virustotal data="url"></uq-virustotal>
<uq-googlesafebrowsing data="url"></uq-googlesafebrowsing>
@ -12,9 +12,9 @@
<uq-whois data="url"></uq-whois><br>
<div ng-repeat="ip in ipv4">
<accordion>
<accordion-group is-open=true>
<accordion-heading>{{ip}}</accordion-heading>
<uib-accordion>
<div uib-accordion-group is-open=true>
<uib-accordion-heading>{{ip}}</uib-accordion-heading>
<uq-phishtank data="ip"></uq-phishtank>
<!-- <li><uq-virustotal data="ip"></uq-virustotal></li> -->
<uq-bgpranking data="ip"></uq-bgpranking>
@ -23,14 +23,14 @@
<uq-psslcircl data="ip"></uq-psslcircl>
<uq-ticket data="ip"></uq-ticket>
<uq-whois data="ip"></uq-whois>
</accordion-group>
</accordion>
</div>
</uib-accordion>
</div>
<div ng-repeat="ip in ipv6">
<accordion>
<accordion-group is-open=true>
<accordion-heading>{{ip}}</accordion-heading>
<uib-accordion>
<div uib-accordion-group is-open=true>
<uib-accordion-heading>{{ip}}</uib-accordion-heading>
<uq-phishtank data="ip"></uq-phishtank>
<!-- <li><uq-virustotal data="ip"></uq-virustotal></li> -->
<!--<li><uq-bgpranking data="ip"></uq-bgpranking></li>-->
@ -38,10 +38,10 @@
<uq-pdnscircl data="ip"></uq-pdnscircl>
<uq-ticket data="ip"></uq-ticket>
<uq-whois data="ip"></uq-whois>
</accordion-group>
</accordion>
</div>
</uib-accordion>
</div>
</accordion-group>
</accordion>
</div>
</uib-accordion>
{% endraw %}