Add mail notification.

requirements.txt

@@ -1,5 +1,6 @@
 flask
 flask-bootstrap
+flask-mail
 flask-wtf
 rq
 redis

url_abuse_async.py

@@ -400,3 +400,42 @@ def bgpranking(ip):
     response = (ptrr, asn_descr, asn, int(position), int(total), float(rank))
     _cache_set(ip, response, 'bgp')
     return response
+
+
+def _deserialize_cached(entry):
+    to_return = {}
+    h = r_cache.hgetall(entry)
+    for key, value in h.iteritems():
+        to_return[key] = json.loads(value)
+    return to_return
+
+
+def get_url_data(url):
+    data = _deserialize_cached(url)
+    if data.get('dns') is not None:
+        ipv4, ipv6 = data['dns']
+        ip_data = {}
+        if ipv4 is not None:
+            for ip in ipv4:
+                ip_data[ip] = _deserialize_cached(ip)
+        if ipv6 is not None:
+            for ip in ipv6:
+                ip_data[ip] = _deserialize_cached(ip)
+        if len(ip_data) > 0:
+            data.update(ip_data)
+    return {url: data}
+
+
+def get_cached(url):
+    _cache_init()
+    if not enable_cache:
+        return [url]
+    url_data = get_url_data(url)
+    to_return = [url_data]
+    if url_data[url].get('list') is not None:
+        url_redirs = url_data[url]['list']
+        for u in url_redirs:
+            if u == url:
+                continue
+            to_return.append(get_url_data(u))
+    return to_return

web/__init__.py

@@ -1,7 +1,8 @@
 import json
 import os
 
-from flask import Flask, render_template, request, Response, redirect, url_for
+from flask import Flask, render_template, request, Response, redirect, url_for, flash
+from flask_mail import Mail, Message
 from flask_bootstrap import Bootstrap
 from flask_wtf import Form
 from wtforms import StringField, SubmitField
@@ -20,7 +21,8 @@ import ConfigParser
 # from pyfaup.faup import Faup
 from proxied import ReverseProxied
 from url_abuse_async import is_valid_url, url_list, dns_resolve, phish_query, psslcircl, \
-    vt_query_url, gsb_query, urlquery_query, sphinxsearch, whois, pdnscircl, bgpranking
+    vt_query_url, gsb_query, urlquery_query, sphinxsearch, whois, pdnscircl, bgpranking, \
+    get_cached
 
 config_path = 'config.ini'
 
@@ -70,6 +72,11 @@ def create_app(configfile=None):
     Bootstrap(app)
     q = Queue(connection=conn)
 
+    # Mail Config
+    app.config['MAIL_SERVER'] = 'localhost'
+    app.config['MAIL_PORT'] = 25
+    mail = Mail(app)
+
     app.config['SECRET_KEY'] = 'devkey'
     app.config['BOOTSTRAP_SERVE_LOCAL'] = True
     app.config['configfile'] = config_path
@@ -83,6 +90,12 @@ def create_app(configfile=None):
                   for i in parser.get('abuse', 'ignore').split('\n')
                   if len(i.strip()) > 0]
 
+    def _get_user_ip(request):
+        ip = request.headers.get('X-Forwarded-For')
+        if ip is None:
+            ip = request.remote_addr
+        return ip
+
     @app.route('/', methods=['GET', 'POST'])
     def index():
         form = URLForm()
@@ -138,9 +151,7 @@ def create_app(configfile=None):
     def run_query():
         data = json.loads(request.data)
         url = data["url"]
-        ip = request.headers.get('X-Forwarded-For')
+        ip = _get_user_ip(request)
-        if ip is None:
-            ip = request.remote_addr
         app.logger.info('{} {}'.format(ip, url))
         is_valid = q.enqueue_call(func=is_valid_url, args=(url,), result_ttl=500)
         return is_valid.get_id()
@@ -257,4 +268,22 @@ def create_app(configfile=None):
                                                  query,), result_ttl=500)
         return u.get_id()
 
+    @app.route('/get_cache/<path:url>')
+    def get_cache(url):
+        data = get_cached(url)
+        dumped = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))
+        return dumped
+
+    @app.route('/submit/<path:url>')
+    def send_mail(url):
+        ip = _get_user_ip(request)
+        data = get_cached(url)
+        dumped = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))
+        msg = Message('URL Abuse report from ' + ip, sender='urlabuse@circl.lu',
+                      recipients=["info@circl.lu"])
+        msg.body = dumped
+        mail.send(msg)
+        flash('Mail successfully sent to CIRCL.')
+        return redirect(url_for('index'))
+
     return app

web/templates/index.html

@@ -19,6 +19,15 @@
     <h1>URL Abuse testing form</h1>
     <h3><div align="center">URL Abuse is a public CIRCL service to review URL.<br /><a target="_blank" href="https://www.circl.lu/services/urlabuse/">For more information about the service</a></div></h3>
   <br/>
+  {% with messages = get_flashed_messages() %}
+    {% if messages %}
+        <div class="flashes alert">
+            {% for message in messages %}
+            <center>{{ message }}</center>
+            {% endfor %}
+        </div>
+    {% endif %}
+  {% endwith %}
   <br/>
   <form class="form form-horizontal" method="post" role="form" ng-submit="getResults()">
     {{ form.hidden_tag() }}
@@ -46,6 +55,11 @@
           <center> <img src="{{ url_for('static', filename='ajax-loader.gif') }}"/> </center>
           {% raw %}
       </div>
+      <div ng-show="urls">
+        <br/>
+        <center><a href="submit/{{ query_url }}">Send report to CIRCL</a></center>
+        <br/>
+      </div>
       <div ng-repeat="url in urls">
         <div uq-urlreport="url"></div>
       </div>