diff --git a/README.md b/README.md index 77870b40..0ee00615 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ The default [installing_deps.sh](./installing_deps.sh) is for Debian and Ubuntu There is also a [Travis file](.travis.yml) used for automating the installation that can be used to build and install AIL on other systems. -Requirement: +Requirement: - Python 3.5+ Installation Notes @@ -165,7 +165,6 @@ Tor hidden service crawler Trending charts --------------- -![Trending-Web](./doc/screenshots/trending-web.png?raw=true "AIL framework webtrending") ![Trending-Modules](./doc/screenshots/trending-module.png?raw=true "AIL framework modulestrending") Extracted encoded files from pastes @@ -199,15 +198,10 @@ Sentiment analysis ![Sentiment](./doc/screenshots/sentiment.png?raw=true "AIL framework sentimentanalysis") -Terms manager and occurence +Terms tracker --------------------------- -![Term-Manager](./doc/screenshots/terms-manager.png?raw=true "AIL framework termManager") - -### Top terms - -![Term-Top](./doc/screenshots/terms-top.png?raw=true "AIL framework termTop") -![Term-Plot](./doc/screenshots/terms-plot.png?raw=true "AIL framework termPlot") +![Term-tracker](./doc/screenshots/terms-tracker.png?raw=true "AIL framework termManager") [AIL framework screencast](https://www.youtube.com/watch?v=1_ZrZkRKmNo) @@ -241,4 +235,3 @@ License You should have received a copy of the GNU Affero General Public License along with this program. If not, see . ``` - diff --git a/doc/screenshots/term-tracker.png b/doc/screenshots/term-tracker.png new file mode 100644 index 00000000..f7284da8 Binary files /dev/null and b/doc/screenshots/term-tracker.png differ diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index adc235d7..223f991b 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -150,105 +150,106 @@ def save_tag_to_auto_push(list_tag): # ============ ROUTES ============ + # TODO: remove + clean -@terms.route("/terms_plot_tool/") -@login_required -@login_read_only -def terms_plot_tool(): - term = request.args.get('term') - if term is not None: - return render_template("terms_plot_tool.html", term=term) - else: - return render_template("terms_plot_tool.html", term="") +# @terms.route("/terms_plot_tool/") +# @login_required +# @login_read_only +# def terms_plot_tool(): +# term = request.args.get('term') +# if term is not None: +# return render_template("terms_plot_tool.html", term=term) +# else: +# return render_template("terms_plot_tool.html", term="") +# +# +# @terms.route("/terms_plot_tool_data/") +# @login_required +# @login_read_only +# def terms_plot_tool_data(): +# oneDay = 60*60*24 +# range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0; +# range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0) +# range_start = calendar.timegm(range_start.timetuple()) +# range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0; +# range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0) +# range_end = calendar.timegm(range_end.timetuple()) +# term = request.args.get('term') +# +# per_paste = request.args.get('per_paste') +# if per_paste == "1" or per_paste is None: +# per_paste = "per_paste_" +# else: +# per_paste = "" +# +# if term is None: +# return "None" +# +# else: +# value_range = [] +# for timestamp in range(range_start, range_end+oneDay, oneDay): +# value = r_serv_term.hget(per_paste+str(timestamp), term) +# curr_value_range = int(value) if value is not None else 0 +# value_range.append([timestamp, curr_value_range]) +# value_range.insert(0,term) +# return jsonify(value_range) +# - -@terms.route("/terms_plot_tool_data/") -@login_required -@login_read_only -def terms_plot_tool_data(): - oneDay = 60*60*24 - range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0; - range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0) - range_start = calendar.timegm(range_start.timetuple()) - range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0; - range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0) - range_end = calendar.timegm(range_end.timetuple()) - term = request.args.get('term') - - per_paste = request.args.get('per_paste') - if per_paste == "1" or per_paste is None: - per_paste = "per_paste_" - else: - per_paste = "" - - if term is None: - return "None" - - else: - value_range = [] - for timestamp in range(range_start, range_end+oneDay, oneDay): - value = r_serv_term.hget(per_paste+str(timestamp), term) - curr_value_range = int(value) if value is not None else 0 - value_range.append([timestamp, curr_value_range]) - value_range.insert(0,term) - return jsonify(value_range) - - -@terms.route("/terms_plot_top/") -@login_required -@login_read_only -def terms_plot_top(): - per_paste = request.args.get('per_paste') - per_paste = per_paste if per_paste is not None else 1 - return render_template("terms_plot_top.html", per_paste=per_paste) - - -@terms.route("/terms_plot_top_data/") -@login_required -@login_read_only -def terms_plot_top_data(): - oneDay = 60*60*24 - today = datetime.datetime.now() - today = today.replace(hour=0, minute=0, second=0, microsecond=0) - today_timestamp = calendar.timegm(today.timetuple()) - - per_paste = request.args.get('per_paste') - if per_paste == "1" or per_paste is None: - per_paste = "per_paste_" - else: - per_paste = "" - - set_day = per_paste + "TopTermFreq_set_day_" + str(today_timestamp) - set_week = per_paste + "TopTermFreq_set_week"; - set_month = per_paste + "TopTermFreq_set_month"; - - the_set = per_paste + request.args.get('set') - num_day = int(request.args.get('num_day')) - - if the_set is None: - return "None" - else: - to_return = [] - if "TopTermFreq_set_day" in the_set: - the_set += "_" + str(today_timestamp) - - for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20): - position = {} - position['day'] = r_serv_term.zrevrank(set_day, term) - position['day'] = position['day']+1 if position['day'] is not None else "<20" - position['week'] = r_serv_term.zrevrank(set_week, term) - position['week'] = position['week']+1 if position['week'] is not None else "<20" - position['month'] = r_serv_term.zrevrank(set_month, term) - position['month'] = position['month']+1 if position['month'] is not None else "<20" - value_range = [] - for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay): - value = r_serv_term.hget(per_paste+str(timestamp), term) - curr_value_range = int(value) if value is not None else 0 - value_range.append([timestamp, curr_value_range]) - - to_return.append([term, value_range, tot_value, position]) - - return jsonify(to_return) +# @terms.route("/terms_plot_top/" +# @login_required +# @login_read_only +# def terms_plot_top(): +# per_paste = request.args.get('per_paste') +# per_paste = per_paste if per_paste is not None else 1 +# return render_template("terms_plot_top.html", per_paste=per_paste) +# +# +# @terms.route("/terms_plot_top_data/") +# @login_required +# @login_read_only +# def terms_plot_top_data(): +# oneDay = 60*60*24 +# today = datetime.datetime.now() +# today = today.replace(hour=0, minute=0, second=0, microsecond=0) +# today_timestamp = calendar.timegm(today.timetuple()) +# +# per_paste = request.args.get('per_paste') +# if per_paste == "1" or per_paste is None: +# per_paste = "per_paste_" +# else: +# per_paste = "" +# +# set_day = per_paste + "TopTermFreq_set_day_" + str(today_timestamp) +# set_week = per_paste + "TopTermFreq_set_week"; +# set_month = per_paste + "TopTermFreq_set_month"; +# +# the_set = per_paste + request.args.get('set') +# num_day = int(request.args.get('num_day')) +# +# if the_set is None: +# return "None" +# else: +# to_return = [] +# if "TopTermFreq_set_day" in the_set: +# the_set += "_" + str(today_timestamp) +# +# for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20): +# position = {} +# position['day'] = r_serv_term.zrevrank(set_day, term) +# position['day'] = position['day']+1 if position['day'] is not None else "<20" +# position['week'] = r_serv_term.zrevrank(set_week, term) +# position['week'] = position['week']+1 if position['week'] is not None else "<20" +# position['month'] = r_serv_term.zrevrank(set_month, term) +# position['month'] = position['month']+1 if position['month'] is not None else "<20" +# value_range = [] +# for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay): +# value = r_serv_term.hget(per_paste+str(timestamp), term) +# curr_value_range = int(value) if value is not None else 0 +# value_range.append([timestamp, curr_value_range]) +# +# to_return.append([term, value_range, tot_value, position]) +# +# return jsonify(to_return) @terms.route("/credentials_tracker/") diff --git a/var/www/modules/terms/templates/header_terms.html b/var/www/modules/terms/templates/header_terms.html index 90e0f6e6..37847a3a 100644 --- a/var/www/modules/terms/templates/header_terms.html +++ b/var/www/modules/terms/templates/header_terms.html @@ -2,7 +2,5 @@ diff --git a/var/www/modules/trendingcharts/templates/header_trendingcharts.html b/var/www/modules/trendingcharts/templates/header_trendingcharts.html deleted file mode 100644 index 605d4fa9..00000000 --- a/var/www/modules/trendingcharts/templates/header_trendingcharts.html +++ /dev/null @@ -1 +0,0 @@ -
  • Trending charts