#!/usr/bin/env python2 # -*-coding:UTF-8 -* import redis import ConfigParser import json from flask import Flask, render_template, jsonify, request import flask import os import sys sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Paste # CONFIG # configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') if not os.path.exists(configfile): raise Exception('Unable to find the configuration file. \ Did you set environment variables? \ Or activate the virtualenv.') cfg = ConfigParser.ConfigParser() cfg.read(configfile) max_preview_char = int(cfg.get("Flask", "max_preview_char")) max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) index_prev = 0 # used if the user want to load more paste content # REDIS # r_serv = redis.StrictRedis( host=cfg.get("Redis_Queues", "host"), port=cfg.getint("Redis_Queues", "port"), db=cfg.getint("Redis_Queues", "db")) r_serv_log = redis.StrictRedis( host=cfg.get("Redis_Log", "host"), port=cfg.getint("Redis_Log", "port"), db=cfg.getint("Redis_Log", "db")) app = Flask(__name__, static_url_path='/static/') def event_stream(): pubsub = r_serv_log.pubsub() pubsub.psubscribe("Script" + '.*') for msg in pubsub.listen(): level = msg['channel'].split('.')[1] if msg['type'] == 'pmessage' and level != "DEBUG": yield 'data: %s\n\n' % json.dumps(msg) def get_queues(r): # We may want to put the llen in a pipeline to do only one query. return [(queue, int(card)) for queue, card in r.hgetall("queues").iteritems()] def list_len(s): return len(s) app.jinja_env.filters['list_len'] = list_len @app.route("/_logs") def logs(): return flask.Response(event_stream(), mimetype="text/event-stream") @app.route("/_stuff", methods=['GET']) def stuff(): return jsonify(row1=get_queues(r_serv)) @app.route("/search", methods=['POST']) def search(): query = request.form['query'] q = [] q.append(query) r = [] c = [] paste_date = [] paste_size = [] # Search from whoosh import index from whoosh.fields import Schema, TEXT, ID schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path")) ix = index.open_dir(indexpath) from whoosh.qparser import QueryParser with ix.searcher() as searcher: query = QueryParser("content", ix.schema).parse(" ".join(q)) results = searcher.search(query, limit=None) for x in results: r.append(x.items()[0][1]) paste = Paste.Paste(x.items()[0][1]) content = paste.get_p_content().decode('utf8', 'ignore') content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 c.append(content[0:content_range]) paste_date.append(paste._get_p_date()) paste_size.append(paste._get_p_size()) return render_template("search.html", r=r, c=c, paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal) @app.route("/") def index(): return render_template("index.html") @app.route("/monitoring/") def monitoring(): for queue in r_serv.smembers("queues"): return render_template("Queue_live_Monitoring.html", last_value=queue) @app.route("/wordstrending/") def wordstrending(): return render_template("Wordstrending.html") @app.route("/protocolstrending/") def protocolstrending(): return render_template("Protocolstrending.html") @app.route("/tldstrending/") def tldstrending(): return render_template("Tldstrending.html") @app.route("/showsavedpaste/") def showsavedpaste(): requested_path = request.args.get('paste', '') paste = Paste.Paste(requested_path) p_date = str(paste._get_p_date()) p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] p_source = paste.p_source p_encoding = paste._get_p_encoding() p_language = paste._get_p_language() p_size = paste.p_size p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content().decode('utf-8', 'ignore') return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content) @app.route("/showpreviewpaste/") def showpreviewpaste(): requested_path = request.args.get('paste', '') paste = Paste.Paste(requested_path) p_date = str(paste._get_p_date()) p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] p_source = paste.p_source p_encoding = paste._get_p_encoding() p_language = paste._get_p_language() p_size = paste.p_size p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() #p_content = paste.get_p_content()[0:max_preview_modal].decode('utf-8', 'ignore') p_content = paste.get_p_content().decode('utf-8', 'ignore') p_content = p_content[0:max_preview_modal] p_initsize = len(p_content) return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=p_initsize) @app.route("/getmoredata/") def getmoredata(): requested_path = request.args.get('paste', '') paste = Paste.Paste(requested_path) p_content = paste.get_p_content().decode('utf-8', 'ignore') to_return = p_content[max_preview_modal:] return to_return if __name__ == "__main__": app.run(host='0.0.0.0', port=7000, threaded=True)