Merge branch 'master' of github.com:CIRCL/url-abuse

url_abuse_async.py

@@ -8,7 +8,11 @@
 from datetime import date
 import json
 import redis
-import urllib
+try:
+    from urllib.parse import quote
+except ImportError:
+    from urllib import quote
+
 from pyfaup.faup import Faup
 import socket
 import dns.resolver
@@ -255,7 +259,7 @@ def phish_query(url, key, query):
     cached = _cache_get(query, 'phishtank')
     if cached is not None:
         return cached
-    postfields = {'url': urllib.quote(query), 'format': 'json', 'app_key': key}
+    postfields = {'url': quote(query), 'format': 'json', 'app_key': key}
     response = requests.post(url, data=postfields)
     res = response.json()
     if res["meta"]["status"] == "success":
@@ -355,7 +359,7 @@ def process_emails(emails, ignorelist, replacelist):
             if re.search(ignorelist_entry, mail, re.I):
                 if mail in to_return:
                     to_return.remove(mail)
-        for k, v in replacelist.iteritems():
+        for k, v in list(replacelist.items()):
             if re.search(k, mail, re.I):
                 if k in to_return:
                     to_return.remove(k)
@@ -372,9 +376,9 @@ def whois(server, port, domain, ignorelist, replacelist):
     try:
         s.connect((server, port))
     except Exception:
-        print "Connection problems - check WHOIS server"
-        print "WHOIS request while problem occurred: " + domain
-        print "WHOIS server: {}:{}".format(server, port)
+        print("Connection problems - check WHOIS server")
+        print(("WHOIS request while problem occurred: ", domain))
+        print(("WHOIS server: {}:{}".format(server, port)))
         sys.exit(0)
     if domain.startswith('http'):
         fex = Faup()
@@ -451,7 +455,7 @@ def bgpranking(ip):
 def _deserialize_cached(entry):
     to_return = {}
     h = r_cache.hgetall(entry)
-    for key, value in h.iteritems():
+    for key, value in list(h.items()):
         to_return[key] = json.loads(value)
     return to_return
 

web/__init__.py

@@ -1,7 +1,7 @@
 import json
 import os
 
-from flask import Flask, render_template, request, Response, redirect, url_for, jsonify
+from flask import Flask, render_template, request, Response, redirect, url_for
 from flask_mail import Mail, Message
 from flask_bootstrap import Bootstrap
 from flask_wtf import Form
@@ -17,9 +17,12 @@ from rq import Queue
 from rq.job import Job
 from worker import conn
 
-import ConfigParser
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
 # from pyfaup.faup import Faup
-from proxied import ReverseProxied
+from .proxied import ReverseProxied
 from url_abuse_async import is_valid_url, url_list, dns_resolve, phish_query, psslcircl, \
     vt_query_url, gsb_query, urlquery_query, sphinxsearch, whois, pdnscircl, bgpranking, \
     cached, get_mail_sent, set_mail_sent, get_submissions
@@ -81,7 +84,7 @@ def create_app(configfile=None):
     app.config['BOOTSTRAP_SERVE_LOCAL'] = True
     app.config['configfile'] = config_path
 
-    parser = ConfigParser.SafeConfigParser()
+    parser = configparser.SafeConfigParser()
     parser.read(app.config['configfile'])
 
     replacelist = make_dict(parser, 'replacelist')
@@ -283,7 +286,7 @@ def create_app(configfile=None):
         to_return = ''
         all_mails = set()
         for entry in data:
-            for url, info in entry.iteritems():
+            for url, info in list(entry.items()):
                 to_return += '\n{}\n'.format(url)
                 if info.get('whois'):
                     all_mails.update(info.get('whois'))

worker.py

@@ -14,6 +14,6 @@ conn = redis.from_url(redis_url)
 
 if __name__ == '__main__':
     with Connection(conn):
-        worker = Worker(map(Queue, listen))
+        worker = Worker(list(map(Queue, listen)))
         worker.work()