Improve send mail support

- Automatically send a mail after a certain threshold
- Do not send more than one email per day for the same URL
travis
Raphaël Vinot 2015-03-05 17:15:55 +01:00
parent c1bf7ede52
commit 53131f3338
2 changed files with 59 additions and 9 deletions

View File

@ -5,6 +5,7 @@
# Copyright (C) 2014 CIRCL Computer Incident Response Center Luxembourg (SMILE gie) # Copyright (C) 2014 CIRCL Computer Incident Response Center Luxembourg (SMILE gie)
# #
from datetime import date
import json import json
import redis import redis
import urllib import urllib
@ -66,8 +67,43 @@ def to_bool(s):
return s.lower() in ('1', 'true', 'yes', 'on') return s.lower() in ('1', 'true', 'yes', 'on')
def get_submissions(url, day=None):
_cache_init()
if enable_cache:
if day is None:
day = date.today().isoformat()
else:
day = day.isoformat()
key = date.today().isoformat() + '_submissions'
return r_cache.zscore(key, url)
def get_mail_sent(url, day=None):
_cache_init()
if enable_cache:
if day is None:
day = date.today().isoformat()
else:
day = day.isoformat()
key = date.today().isoformat() + '_mails'
return r_cache.sismember(key, url)
def set_mail_sent(url, day=None):
_cache_init()
if enable_cache:
if day is None:
day = date.today().isoformat()
else:
day = day.isoformat()
key = date.today().isoformat() + '_mails'
return r_cache.sadd(key, url)
def is_valid_url(url): def is_valid_url(url):
cached = _cache_get(url, 'valid') cached = _cache_get(url, 'valid')
key = date.today().isoformat() + '_submissions'
r_cache.zincrby(key, url)
if cached is not None: if cached is not None:
return cached return cached
fex = Faup() fex = Faup()

View File

@ -22,7 +22,7 @@ import ConfigParser
from proxied import ReverseProxied from proxied import ReverseProxied
from url_abuse_async import is_valid_url, url_list, dns_resolve, phish_query, psslcircl, \ from url_abuse_async import is_valid_url, url_list, dns_resolve, phish_query, psslcircl, \
vt_query_url, gsb_query, urlquery_query, sphinxsearch, whois, pdnscircl, bgpranking, \ vt_query_url, gsb_query, urlquery_query, sphinxsearch, whois, pdnscircl, bgpranking, \
get_cached get_cached, get_mail_sent, set_mail_sent, get_submissions
config_path = 'config.ini' config_path = 'config.ini'
@ -89,6 +89,7 @@ def create_app(configfile=None):
ignorelist = [i.strip() ignorelist = [i.strip()
for i in parser.get('abuse', 'ignore').split('\n') for i in parser.get('abuse', 'ignore').split('\n')
if len(i.strip()) > 0] if len(i.strip()) > 0]
autosend_threshold = 5
def _get_user_ip(request): def _get_user_ip(request):
ip = request.headers.get('X-Forwarded-For') ip = request.headers.get('X-Forwarded-For')
@ -153,6 +154,8 @@ def create_app(configfile=None):
url = data["url"] url = data["url"]
ip = _get_user_ip(request) ip = _get_user_ip(request)
app.logger.info('{} {}'.format(ip, url)) app.logger.info('{} {}'.format(ip, url))
if get_submissions(url) >= autosend_threshold:
send(url, '', True)
is_valid = q.enqueue_call(func=is_valid_url, args=(url,), result_ttl=500) is_valid = q.enqueue_call(func=is_valid_url, args=(url,), result_ttl=500)
return is_valid.get_id() return is_valid.get_id()
@ -274,16 +277,27 @@ def create_app(configfile=None):
dumped = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')) dumped = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))
return dumped return dumped
def send(url, ip='', autosend=False):
if not get_mail_sent(url):
print 'Send mail'
set_mail_sent(url)
data = get_cached(url)
if not autosend:
subject = 'URL Abuse report from ' + ip
else:
subject = 'URL Abuse report sent automatically'
msg = Message(subject, sender='urlabuse@circl.lu', recipients=["info@circl.lu"])
msg.body = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))
mail.send(msg)
@app.route('/submit/<path:url>') @app.route('/submit/<path:url>')
def send_mail(url): def send_mail(url):
ip = _get_user_ip(request) if get_mail_sent(url):
data = get_cached(url) flash('Mail already sent to CIRCL.')
dumped = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')) else:
msg = Message('URL Abuse report from ' + ip, sender='urlabuse@circl.lu', ip = _get_user_ip(request)
recipients=["info@circl.lu"]) send(url, ip)
msg.body = dumped flash('Mail successfully sent to CIRCL.')
mail.send(msg)
flash('Mail successfully sent to CIRCL.')
return redirect(url_for('index')) return redirect(url_for('index'))
return app return app