Merge branch 'master' into tags_v2

README.md

@@ -90,7 +90,7 @@ The default [installing_deps.sh](./installing_deps.sh) is for Debian and Ubuntu
 
 There is also a [Travis file](.travis.yml) used for automating the installation that can be used to build and install AIL on other systems.
 
-Requirement: 
+Requirement:
 - Python 3.5+
 
 Installation Notes
@@ -165,7 +165,6 @@ Tor hidden service crawler
 Trending charts
 ---------------
 
-![Trending-Web](./doc/screenshots/trending-web.png?raw=true "AIL framework webtrending")
 ![Trending-Modules](./doc/screenshots/trending-module.png?raw=true "AIL framework modulestrending")
 
 Extracted encoded files from pastes
@@ -199,15 +198,10 @@ Sentiment analysis
 
 ![Sentiment](./doc/screenshots/sentiment.png?raw=true "AIL framework sentimentanalysis")
 
-Terms manager and occurence
+Terms tracker
 ---------------------------
 
-![Term-Manager](./doc/screenshots/terms-manager.png?raw=true "AIL framework termManager")
+![Term-tracker](./doc/screenshots/terms-tracker.png?raw=true "AIL framework termManager")
-
-### Top terms
-
-![Term-Top](./doc/screenshots/terms-top.png?raw=true "AIL framework termTop")
-![Term-Plot](./doc/screenshots/terms-plot.png?raw=true "AIL framework termPlot")
 
 
 [AIL framework screencast](https://www.youtube.com/watch?v=1_ZrZkRKmNo)
@@ -241,4 +235,3 @@ License
     You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ```
-

var/www/modules/terms/Flask_terms.py

@@ -150,105 +150,106 @@ def save_tag_to_auto_push(list_tag):
 
 # ============ ROUTES ============
 
+ # TODO: remove + clean
 
-@terms.route("/terms_plot_tool/")
+# @terms.route("/terms_plot_tool/")
-@login_required
+# @login_required
-@login_read_only
+# @login_read_only
-def terms_plot_tool():
+# def terms_plot_tool():
-    term =  request.args.get('term')
+#     term =  request.args.get('term')
-    if term is not None:
+#     if term is not None:
-        return render_template("terms_plot_tool.html", term=term)
+#         return render_template("terms_plot_tool.html", term=term)
-    else:
+#     else:
-        return render_template("terms_plot_tool.html", term="")
+#         return render_template("terms_plot_tool.html", term="")
+#
+#
+# @terms.route("/terms_plot_tool_data/")
+# @login_required
+# @login_read_only
+# def terms_plot_tool_data():
+#     oneDay = 60*60*24
+#     range_start =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
+#     range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
+#     range_start = calendar.timegm(range_start.timetuple())
+#     range_end =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
+#     range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
+#     range_end = calendar.timegm(range_end.timetuple())
+#     term =  request.args.get('term')
+#
+#     per_paste = request.args.get('per_paste')
+#     if per_paste == "1" or per_paste is None:
+#         per_paste = "per_paste_"
+#     else:
+#         per_paste = ""
+#
+#     if term is None:
+#         return "None"
+#
+#     else:
+#         value_range = []
+#         for timestamp in range(range_start, range_end+oneDay, oneDay):
+#             value = r_serv_term.hget(per_paste+str(timestamp), term)
+#             curr_value_range = int(value) if value is not None else 0
+#             value_range.append([timestamp, curr_value_range])
+#         value_range.insert(0,term)
+#         return jsonify(value_range)
+#
 
-
+# @terms.route("/terms_plot_top/"
-@terms.route("/terms_plot_tool_data/")
+# @login_required
-@login_required
+# @login_read_only
-@login_read_only
+# def terms_plot_top():
-def terms_plot_tool_data():
+#     per_paste = request.args.get('per_paste')
-    oneDay = 60*60*24
+#     per_paste = per_paste if per_paste is not None else 1
-    range_start =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
+#     return render_template("terms_plot_top.html", per_paste=per_paste)
-    range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
+# 
-    range_start = calendar.timegm(range_start.timetuple())
+#
-    range_end =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
+# @terms.route("/terms_plot_top_data/")
-    range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
+# @login_required
-    range_end = calendar.timegm(range_end.timetuple())
+# @login_read_only
-    term =  request.args.get('term')
+# def terms_plot_top_data():
-
+#     oneDay = 60*60*24
-    per_paste = request.args.get('per_paste')
+#     today = datetime.datetime.now()
-    if per_paste == "1" or per_paste is None:
+#     today = today.replace(hour=0, minute=0, second=0, microsecond=0)
-        per_paste = "per_paste_"
+#     today_timestamp = calendar.timegm(today.timetuple())
-    else:
+#
-        per_paste = ""
+#     per_paste = request.args.get('per_paste')
-
+#     if per_paste == "1" or per_paste is None:
-    if term is None:
+#         per_paste = "per_paste_"
-        return "None"
+#     else:
-
+#         per_paste = ""
-    else:
+#
-        value_range = []
+#     set_day = per_paste + "TopTermFreq_set_day_" + str(today_timestamp)
-        for timestamp in range(range_start, range_end+oneDay, oneDay):
+#     set_week = per_paste + "TopTermFreq_set_week";
-            value = r_serv_term.hget(per_paste+str(timestamp), term)
+#     set_month = per_paste + "TopTermFreq_set_month";
-            curr_value_range = int(value) if value is not None else 0
+#
-            value_range.append([timestamp, curr_value_range])
+#     the_set = per_paste + request.args.get('set')
-        value_range.insert(0,term)
+#     num_day = int(request.args.get('num_day'))
-        return jsonify(value_range)
+#
-
+#     if the_set is None:
-
+#         return "None"
-@terms.route("/terms_plot_top/")
+#     else:
-@login_required
+#         to_return = []
-@login_read_only
+#         if "TopTermFreq_set_day" in the_set:
-def terms_plot_top():
+#             the_set += "_" + str(today_timestamp)
-    per_paste = request.args.get('per_paste')
+#
-    per_paste = per_paste if per_paste is not None else 1
+#         for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
-    return render_template("terms_plot_top.html", per_paste=per_paste)
+#             position = {}
-
+#             position['day'] = r_serv_term.zrevrank(set_day, term)
-
+#             position['day'] = position['day']+1 if position['day'] is not None else "<20"
-@terms.route("/terms_plot_top_data/")
+#             position['week'] = r_serv_term.zrevrank(set_week, term)
-@login_required
+#             position['week'] = position['week']+1 if position['week'] is not None else "<20"
-@login_read_only
+#             position['month'] = r_serv_term.zrevrank(set_month, term)
-def terms_plot_top_data():
+#             position['month'] = position['month']+1 if position['month'] is not None else "<20"
-    oneDay = 60*60*24
+#             value_range = []
-    today = datetime.datetime.now()
+#             for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
-    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
+#                 value = r_serv_term.hget(per_paste+str(timestamp), term)
-    today_timestamp = calendar.timegm(today.timetuple())
+#                 curr_value_range = int(value) if value is not None else 0
-
+#                 value_range.append([timestamp, curr_value_range])
-    per_paste = request.args.get('per_paste')
+#
-    if per_paste == "1" or per_paste is None:
+#             to_return.append([term, value_range, tot_value, position])
-        per_paste = "per_paste_"
+#
-    else:
+#         return jsonify(to_return)
-        per_paste = ""
-
-    set_day = per_paste + "TopTermFreq_set_day_" + str(today_timestamp)
-    set_week = per_paste + "TopTermFreq_set_week";
-    set_month = per_paste + "TopTermFreq_set_month";
-
-    the_set = per_paste + request.args.get('set')
-    num_day = int(request.args.get('num_day'))
-
-    if the_set is None:
-        return "None"
-    else:
-        to_return = []
-        if "TopTermFreq_set_day" in the_set:
-            the_set += "_" + str(today_timestamp)
-
-        for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
-            position = {}
-            position['day'] = r_serv_term.zrevrank(set_day, term)
-            position['day'] = position['day']+1 if position['day'] is not None else "<20"
-            position['week'] = r_serv_term.zrevrank(set_week, term)
-            position['week'] = position['week']+1 if position['week'] is not None else "<20"
-            position['month'] = r_serv_term.zrevrank(set_month, term)
-            position['month'] = position['month']+1 if position['month'] is not None else "<20"
-            value_range = []
-            for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
-                value = r_serv_term.hget(per_paste+str(timestamp), term)
-                curr_value_range = int(value) if value is not None else 0
-                value_range.append([timestamp, curr_value_range])
-
-            to_return.append([term, value_range, tot_value, position])
-
-        return jsonify(to_return)
 
 
 @terms.route("/credentials_tracker/")

var/www/modules/terms/templates/header_terms.html

@@ -2,7 +2,5 @@
     <span class="caret"></span></a>
     <ul class="dropdown-menu">
       <li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
-      <li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
-      <li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
     </ul>
 </li>

var/www/modules/trendingcharts/templates/header_trendingcharts.html

@@ -1 +0,0 @@
-<li id='page-trendingchart'><a href="{{ url_for('trendings.trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>