mirror of https://github.com/CIRCL/url-abuse
chg: Sync with master
Raphaël Vinot
parent
62c1328075
commit
150a908aac
|
|
@ -5,13 +5,13 @@ set -x
|
|||
|
||||
DEST_DIR="web/static/"
|
||||
|
||||
ANGULAR='1.4.5'
|
||||
ANGULAR_BOOTSTRAP='0.13.4'
|
||||
ANGULAR='1.7.4'
|
||||
ANGULAR_BOOTSTRAP='2.5.0'
|
||||
|
||||
wget https://ajax.googleapis.com/ajax/libs/angularjs/${ANGULAR}/angular.min.js -O ${DEST_DIR}/angular.min.js
|
||||
wget https://angular-ui.github.io/bootstrap/ui-bootstrap-tpls-${ANGULAR_BOOTSTRAP}.min.js -O ${DEST_DIR}/ui-bootstrap-tpls.min.js
|
||||
|
||||
wget https://sphinxsearch.googlecode.com/svn/trunk/api/sphinxapi.py -O sphinxapi.py
|
||||
wget https://raw.githubusercontent.com/sphinxsearch/sphinx/master/api/sphinxapi.py -O sphinxapi.py
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -20,15 +20,15 @@ import re
|
|||
import sys
|
||||
import logging
|
||||
from pypdns import PyPDNS
|
||||
import bgpranking_web
|
||||
import urlquery
|
||||
# import bgpranking_web
|
||||
# import urlquery
|
||||
from pypssl import PyPSSL
|
||||
from pyeupi import PyEUPI
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
try:
|
||||
import sphinxapi
|
||||
# import sphinxapi
|
||||
sphinx = True
|
||||
except:
|
||||
sphinx = False
|
||||
|
|
@ -40,7 +40,7 @@ r_cache = None
|
|||
def _cache_init(host='localhost', port=6334, db=1):
|
||||
global r_cache
|
||||
if enable_cache and r_cache is None:
|
||||
r_cache = redis.Redis(host, port, db=db)
|
||||
r_cache = redis.Redis(host, port, db=db, decode_responses=True)
|
||||
|
||||
|
||||
def _cache_set(key, value, field=None):
|
||||
|
|
@ -128,7 +128,12 @@ def is_valid_url(url):
|
|||
|
||||
|
||||
def is_ip(host):
|
||||
if ':' in host:
|
||||
if isinstance(host, bytes):
|
||||
to_search = b':'
|
||||
else:
|
||||
to_search = ':'
|
||||
|
||||
if to_search in host:
|
||||
try:
|
||||
socket.inet_pton(socket.AF_INET6, host)
|
||||
return True
|
||||
|
|
@ -173,8 +178,11 @@ def get_urls(url, depth=1):
|
|||
out = result["content"].split(";")
|
||||
if len(out) == 2:
|
||||
wait, text = out
|
||||
a, url = text.split('=', 1)
|
||||
return url.strip()
|
||||
try:
|
||||
a, url = text.split('=', 1)
|
||||
return url.strip()
|
||||
except:
|
||||
print(text)
|
||||
return None
|
||||
|
||||
resolve, reason = try_resolve(fex, url)
|
||||
|
|
@ -240,7 +248,7 @@ def dns_resolve(url):
|
|||
return cached
|
||||
fex = Faup()
|
||||
fex.decode(url)
|
||||
host = fex.get_host().lower()
|
||||
host = fex.get_host().decode().lower()
|
||||
ipv4 = None
|
||||
ipv6 = None
|
||||
if is_ip(host):
|
||||
|
|
@ -291,6 +299,9 @@ def phish_query(url, key, query):
|
|||
|
||||
|
||||
def sphinxsearch(server, port, url, query):
|
||||
# WARNING: too dangerous to have on the public interface
|
||||
return ''
|
||||
"""
|
||||
if not sphinx:
|
||||
return None
|
||||
cached = _cache_get(query, 'sphinx')
|
||||
|
|
@ -310,6 +321,8 @@ def sphinxsearch(server, port, url, query):
|
|||
_cache_set(query, result, 'sphinx')
|
||||
return result
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def vt_query_url(url, url_up, key, query, upload=True):
|
||||
cached = _cache_get(query, 'vt')
|
||||
|
|
@ -344,6 +357,7 @@ def gsb_query(url, query):
|
|||
|
||||
|
||||
def urlquery_query(url, key, query):
|
||||
return None
|
||||
cached = _cache_get(query, 'urlquery')
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
|
@ -400,15 +414,15 @@ def whois(server, port, domain, ignorelist, replacelist):
|
|||
d = fex.get_domain().lower()
|
||||
else:
|
||||
d = domain
|
||||
s.send(d + "\r\n")
|
||||
response = ''
|
||||
s.send(("{}\r\n".format(d)).encode())
|
||||
response = b''
|
||||
while True:
|
||||
d = s.recv(4096)
|
||||
response += d
|
||||
if d == '':
|
||||
if d == b'':
|
||||
break
|
||||
s.close()
|
||||
match = re.findall(r'[\w\.-]+@[\w\.-]+', response)
|
||||
match = re.findall(r'[\w\.-]+@[\w\.-]+', response.decode())
|
||||
emails = process_emails(match, ignorelist, replacelist)
|
||||
if len(emails) == 0:
|
||||
return None
|
||||
|
|
@ -469,6 +483,7 @@ def eupi(url, key, q):
|
|||
|
||||
|
||||
def bgpranking(ip):
|
||||
return None, None, None, None, None, None
|
||||
cached = _cache_get(ip, 'bgp')
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
|
@ -481,7 +496,19 @@ def bgpranking(ip):
|
|||
position, total = bgpranking_web.cached_position(asn)
|
||||
asn_descr = rank_info[1]
|
||||
rank = rank_info[-1]
|
||||
response = (ptrr, asn_descr, asn, int(position), int(total), float(rank))
|
||||
if position:
|
||||
position = int(position)
|
||||
else:
|
||||
position = -1
|
||||
if total:
|
||||
total = int(total)
|
||||
else:
|
||||
total = 0
|
||||
if rank:
|
||||
rank = float(rank)
|
||||
else:
|
||||
rank = -1
|
||||
response = (ptrr, asn_descr, asn, position, total, rank)
|
||||
_cache_set(ip, response, 'bgp')
|
||||
return response
|
||||
|
||||
|
|
|
|||
|
|
@ -153,32 +153,32 @@ def create_app(configfile=None):
|
|||
|
||||
@app.route('/start', methods=['POST'])
|
||||
def run_query():
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
url = data["url"]
|
||||
ip = _get_user_ip(request)
|
||||
app.logger.info('{} {}'.format(ip, url))
|
||||
if get_submissions(url) >= autosend_threshold:
|
||||
if get_submissions(url) and get_submissions(url) >= autosend_threshold:
|
||||
send(url, '', True)
|
||||
is_valid = q.enqueue_call(func=is_valid_url, args=(url,), result_ttl=500)
|
||||
return is_valid.get_id()
|
||||
|
||||
@app.route('/urls', methods=['POST'])
|
||||
def urls():
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
url = data["url"]
|
||||
u = q.enqueue_call(func=url_list, args=(url,), result_ttl=500)
|
||||
return u.get_id()
|
||||
|
||||
@app.route('/resolve', methods=['POST'])
|
||||
def resolve():
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
url = data["url"]
|
||||
u = q.enqueue_call(func=dns_resolve, args=(url,), result_ttl=500)
|
||||
return u.get_id()
|
||||
|
||||
@app.route('/phishtank', methods=['POST'])
|
||||
def phishtank():
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
if not os.path.exists('phishtank.key'):
|
||||
return None
|
||||
url = parser.get("PHISHTANK", "url")
|
||||
|
|
@ -189,7 +189,7 @@ def create_app(configfile=None):
|
|||
|
||||
@app.route('/virustotal_report', methods=['POST'])
|
||||
def vt():
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
if not os.path.exists('virustotal.key'):
|
||||
return None
|
||||
url = parser.get("VIRUSTOTAL", "url_report")
|
||||
|
|
@ -201,7 +201,7 @@ def create_app(configfile=None):
|
|||
|
||||
@app.route('/googlesafebrowsing', methods=['POST'])
|
||||
def gsb():
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
if not os.path.exists('googlesafebrowsing.key'):
|
||||
return None
|
||||
url = parser.get("GOOGLESAFEBROWSING", "url")
|
||||
|
|
@ -213,7 +213,7 @@ def create_app(configfile=None):
|
|||
|
||||
@app.route('/urlquery', methods=['POST'])
|
||||
def urlquery():
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
if not os.path.exists('urlquery.key'):
|
||||
return None
|
||||
url = parser.get("URLQUERY", "url")
|
||||
|
|
@ -226,7 +226,7 @@ def create_app(configfile=None):
|
|||
def ticket():
|
||||
if not request.authorization:
|
||||
return ''
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
server = parser.get("SPHINX", "server")
|
||||
port = int(parser.get("SPHINX", "port"))
|
||||
url = parser.get("ITS", "url")
|
||||
|
|
@ -237,11 +237,11 @@ def create_app(configfile=None):
|
|||
|
||||
@app.route('/whois', methods=['POST'])
|
||||
def whoismail():
|
||||
if not request.authorization:
|
||||
return ''
|
||||
# if not request.authorization:
|
||||
# return ''
|
||||
server = parser.get("WHOIS", "server")
|
||||
port = parser.getint("WHOIS", "port")
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
query = data["query"]
|
||||
u = q.enqueue_call(func=whois, args=(server, port, query, ignorelist, replacelist),
|
||||
result_ttl=500)
|
||||
|
|
@ -249,7 +249,7 @@ def create_app(configfile=None):
|
|||
|
||||
@app.route('/eupi', methods=['POST'])
|
||||
def eu():
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
if not os.path.exists('eupi.key'):
|
||||
return None
|
||||
url = parser.get("EUPI", "url")
|
||||
|
|
@ -262,7 +262,7 @@ def create_app(configfile=None):
|
|||
def dnscircl():
|
||||
url = parser.get("PDNS_CIRCL", "url")
|
||||
user, password = open('pdnscircl.key', 'r').readlines()
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
query = data["query"]
|
||||
u = q.enqueue_call(func=pdnscircl, args=(url, user.strip(), password.strip(),
|
||||
query,), result_ttl=500)
|
||||
|
|
@ -270,7 +270,7 @@ def create_app(configfile=None):
|
|||
|
||||
@app.route('/bgpranking', methods=['POST'])
|
||||
def bgpr():
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
query = data["query"]
|
||||
u = q.enqueue_call(func=bgpranking, args=(query,), result_ttl=500)
|
||||
return u.get_id()
|
||||
|
|
@ -279,7 +279,7 @@ def create_app(configfile=None):
|
|||
def sslcircl():
|
||||
url = parser.get("PSSL_CIRCL", "url")
|
||||
user, password = open('psslcircl.key', 'r').readlines()
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
query = data["query"]
|
||||
u = q.enqueue_call(func=psslcircl, args=(url, user.strip(), password.strip(),
|
||||
query,), result_ttl=500)
|
||||
|
|
@ -287,7 +287,7 @@ def create_app(configfile=None):
|
|||
|
||||
@app.route('/get_cache', methods=['POST'])
|
||||
def get_cache():
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
url = data["query"]
|
||||
data = cached(url)
|
||||
dumped = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))
|
||||
|
|
@ -347,7 +347,7 @@ def create_app(configfile=None):
|
|||
|
||||
@app.route('/submit', methods=['POST'])
|
||||
def send_mail():
|
||||
data = json.loads(request.data)
|
||||
data = json.loads(request.data.decode())
|
||||
url = data["url"]
|
||||
if not get_mail_sent(url):
|
||||
ip = _get_user_ip(request)
|
||||
|
|
|
|||
|
|
@ -26,15 +26,20 @@
|
|||
poller: function myself(jobID, callback) {
|
||||
var timeout = "";
|
||||
// fire another request
|
||||
$http.get('_result/' + jobID).
|
||||
success(function(data, status, headers, config) {
|
||||
if(status === 202) {
|
||||
$http.get('_result/' + jobID.data).
|
||||
then(function(data) {
|
||||
if(data.status === 202) {
|
||||
$log.log(data, status);
|
||||
} else if (status === 200){
|
||||
$log.log(data);
|
||||
} else if (data.status === 200){
|
||||
$log.log(data.data);
|
||||
$timeout.cancel(timeout);
|
||||
callback(angular.fromJson(data));
|
||||
return;
|
||||
if (data.data === "null"){
|
||||
$log.log('Got null data');
|
||||
return;
|
||||
} else {
|
||||
callback(data.data);
|
||||
return;
|
||||
};
|
||||
}
|
||||
// continue to call the poller() function every 2 seconds
|
||||
// until the timout is cancelled
|
||||
|
|
@ -43,8 +48,7 @@
|
|||
},
|
||||
query: function(path, data, callback) {
|
||||
$http.post(path, data).
|
||||
success(callback).
|
||||
error(function(error) {
|
||||
then(callback, function(error) {
|
||||
$log.log(error);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
{% raw %}
|
||||
|
||||
<accordion>
|
||||
<accordion-group is-open=true>
|
||||
<accordion-heading>{{url}}</accordion-heading>
|
||||
<uib-accordion>
|
||||
<div uib-accordion-group is-open=true>
|
||||
<uib-accordion-heading>{{url}}</uib-accordion-heading>
|
||||
<uq-phishtank data="url"></uq-phishtank>
|
||||
<uq-virustotal data="url"></uq-virustotal>
|
||||
<uq-googlesafebrowsing data="url"></uq-googlesafebrowsing>
|
||||
|
|
@ -12,9 +12,9 @@
|
|||
<uq-whois data="url"></uq-whois><br>
|
||||
|
||||
<div ng-repeat="ip in ipv4">
|
||||
<accordion>
|
||||
<accordion-group is-open=true>
|
||||
<accordion-heading>{{ip}}</accordion-heading>
|
||||
<uib-accordion>
|
||||
<div uib-accordion-group is-open=true>
|
||||
<uib-accordion-heading>{{ip}}</uib-accordion-heading>
|
||||
<uq-phishtank data="ip"></uq-phishtank>
|
||||
<!-- <li><uq-virustotal data="ip"></uq-virustotal></li> -->
|
||||
<uq-bgpranking data="ip"></uq-bgpranking>
|
||||
|
|
@ -23,14 +23,14 @@
|
|||
<uq-psslcircl data="ip"></uq-psslcircl>
|
||||
<uq-ticket data="ip"></uq-ticket>
|
||||
<uq-whois data="ip"></uq-whois>
|
||||
</accordion-group>
|
||||
</accordion>
|
||||
</div>
|
||||
</uib-accordion>
|
||||
</div>
|
||||
|
||||
<div ng-repeat="ip in ipv6">
|
||||
<accordion>
|
||||
<accordion-group is-open=true>
|
||||
<accordion-heading>{{ip}}</accordion-heading>
|
||||
<uib-accordion>
|
||||
<div uib-accordion-group is-open=true>
|
||||
<uib-accordion-heading>{{ip}}</uib-accordion-heading>
|
||||
<uq-phishtank data="ip"></uq-phishtank>
|
||||
<!-- <li><uq-virustotal data="ip"></uq-virustotal></li> -->
|
||||
<!--<li><uq-bgpranking data="ip"></uq-bgpranking></li>-->
|
||||
|
|
@ -38,10 +38,10 @@
|
|||
<uq-pdnscircl data="ip"></uq-pdnscircl>
|
||||
<uq-ticket data="ip"></uq-ticket>
|
||||
<uq-whois data="ip"></uq-whois>
|
||||
</accordion-group>
|
||||
</accordion>
|
||||
</div>
|
||||
</uib-accordion>
|
||||
</div>
|
||||
</accordion-group>
|
||||
</accordion>
|
||||
</div>
|
||||
</uib-accordion>
|
||||
|
||||
{% endraw %}
|
||||
|
|
|
|||
Loading…
Reference in New Issue