CIRCL
/
url-abuse
mirror of https://github.com/CIRCL/url-abuse
2
0
Fork 0
Code Issues Releases Wiki Activity

Detect redirect loops and sanitize.

travis
Raphaël Vinot 2015-03-06 16:06:41 +01:00
parent ecd0e443fd
commit a113b4178e
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
url_abuse_async.py
View File

@ -216,7 +216,11 @@ def url_list(url):
cached = _cache_get(url, 'list') cached = _cache_get(url, 'list')
if cached is not None: if cached is not None:
return cached return cached
list_urls = [u for u in get_urls(url) if u is not None] list_urls = []
for u in get_urls(url):
if u is None or u in list_urls:
continue
list_urls.append(u)
_cache_set(url, list_urls, 'list') _cache_set(url, list_urls, 'list')
return list_urls return list_urls