CIRCL
/
url-abuse
mirror of https://github.com/CIRCL/url-abuse
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'master' of github.com:CIRCL/url-abuse

travis
Raphaël Vinot 2015-05-11 17:03:45 +02:00
parent 6ae8ba2d5a fca41e3714
commit 72e1b1ef28
3 changed files with 20 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
url_abuse_async.py
View File

@ -8,7 +8,11 @@
from datetime import date from datetime import date
import json import json
import redis import redis
import urllib try:
from urllib.parse import quote
except ImportError:
from urllib import quote
from pyfaup.faup import Faup from pyfaup.faup import Faup
import socket import socket
import dns.resolver import dns.resolver
@ -255,7 +259,7 @@ def phish_query(url, key, query):
cached = _cache_get(query, 'phishtank') cached = _cache_get(query, 'phishtank')
if cached is not None: if cached is not None:
return cached return cached
postfields = {'url': urllib.quote(query), 'format': 'json', 'app_key': key} postfields = {'url': quote(query), 'format': 'json', 'app_key': key}
response = requests.post(url, data=postfields) response = requests.post(url, data=postfields)
res = response.json() res = response.json()
if res["meta"]["status"] == "success": if res["meta"]["status"] == "success":
@ -355,7 +359,7 @@ def process_emails(emails, ignorelist, replacelist):
if re.search(ignorelist_entry, mail, re.I): if re.search(ignorelist_entry, mail, re.I):
if mail in to_return: if mail in to_return:
to_return.remove(mail) to_return.remove(mail)
for k, v in replacelist.iteritems(): for k, v in list(replacelist.items()):
if re.search(k, mail, re.I): if re.search(k, mail, re.I):
if k in to_return: if k in to_return:
to_return.remove(k) to_return.remove(k)
@ -372,9 +376,9 @@ def whois(server, port, domain, ignorelist, replacelist):
try: try:
s.connect((server, port)) s.connect((server, port))
except Exception: except Exception:
print "Connection problems - check WHOIS server" print("Connection problems - check WHOIS server")
print "WHOIS request while problem occurred: " + domain print(("WHOIS request while problem occurred: ", domain))
print "WHOIS server: {}:{}".format(server, port) print(("WHOIS server: {}:{}".format(server, port)))
sys.exit(0) sys.exit(0)
if domain.startswith('http'): if domain.startswith('http'):
fex = Faup() fex = Faup()
@ -451,7 +455,7 @@ def bgpranking(ip):
def _deserialize_cached(entry): def _deserialize_cached(entry):
to_return = {} to_return = {}
h = r_cache.hgetall(entry) h = r_cache.hgetall(entry)
for key, value in h.iteritems(): for key, value in list(h.items()):
to_return[key] = json.loads(value) to_return[key] = json.loads(value)
return to_return return to_return

13
web/__init__.py
View File

@ -1,7 +1,7 @@
import json import json
import os import os
from flask import Flask, render_template, request, Response, redirect, url_for, jsonify from flask import Flask, render_template, request, Response, redirect, url_for
from flask_mail import Mail, Message from flask_mail import Mail, Message
from flask_bootstrap import Bootstrap from flask_bootstrap import Bootstrap
from flask_wtf import Form from flask_wtf import Form
@ -17,9 +17,12 @@ from rq import Queue
from rq.job import Job from rq.job import Job
from worker import conn from worker import conn
import ConfigParser try:
import configparser
except ImportError:
import ConfigParser as configparser
# from pyfaup.faup import Faup # from pyfaup.faup import Faup
from proxied import ReverseProxied from .proxied import ReverseProxied
from url_abuse_async import is_valid_url, url_list, dns_resolve, phish_query, psslcircl, \ from url_abuse_async import is_valid_url, url_list, dns_resolve, phish_query, psslcircl, \
vt_query_url, gsb_query, urlquery_query, sphinxsearch, whois, pdnscircl, bgpranking, \ vt_query_url, gsb_query, urlquery_query, sphinxsearch, whois, pdnscircl, bgpranking, \
cached, get_mail_sent, set_mail_sent, get_submissions cached, get_mail_sent, set_mail_sent, get_submissions
@ -81,7 +84,7 @@ def create_app(configfile=None):
app.config['BOOTSTRAP_SERVE_LOCAL'] = True app.config['BOOTSTRAP_SERVE_LOCAL'] = True
app.config['configfile'] = config_path app.config['configfile'] = config_path
parser = ConfigParser.SafeConfigParser() parser = configparser.SafeConfigParser()
parser.read(app.config['configfile']) parser.read(app.config['configfile'])
replacelist = make_dict(parser, 'replacelist') replacelist = make_dict(parser, 'replacelist')
@ -283,7 +286,7 @@ def create_app(configfile=None):
to_return = '' to_return = ''
all_mails = set() all_mails = set()
for entry in data: for entry in data:
for url, info in entry.iteritems(): for url, info in list(entry.items()):
to_return += '\n{}\n'.format(url) to_return += '\n{}\n'.format(url)
if info.get('whois'): if info.get('whois'):
all_mails.update(info.get('whois')) all_mails.update(info.get('whois'))

2
worker.py
View File

@ -14,6 +14,6 @@ conn = redis.from_url(redis_url)
if __name__ == '__main__': if __name__ == '__main__':
with Connection(conn): with Connection(conn):
worker = Worker(map(Queue, listen)) worker = Worker(list(map(Queue, listen)))
worker.work() worker.work()