CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'master' into tags_v2

pull/449/head
Terrtia 2020-01-09 15:11:37 +01:00
parent ce3d98d620 5e7a7946ec
commit b4a36d812f
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
5 changed files with 101 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
README.md
View File

@ -90,7 +90,7 @@ The default [installing_deps.sh](./installing_deps.sh) is for Debian and Ubuntu
There is also a [Travis file](.travis.yml) used for automating the installation that can be used to build and install AIL on other systems.
Requirement:
Requirement:
- Python 3.5+
Installation Notes
@ -165,7 +165,6 @@ Tor hidden service crawler
Trending charts
---------------
![Trending-Web](./doc/screenshots/trending-web.png?raw=true "AIL framework webtrending")
![Trending-Modules](./doc/screenshots/trending-module.png?raw=true "AIL framework modulestrending")
Extracted encoded files from pastes
@ -199,15 +198,10 @@ Sentiment analysis
![Sentiment](./doc/screenshots/sentiment.png?raw=true "AIL framework sentimentanalysis")
Terms manager and occurence
Terms tracker
---------------------------
![Term-Manager](./doc/screenshots/terms-manager.png?raw=true "AIL framework termManager")
### Top terms
![Term-Top](./doc/screenshots/terms-top.png?raw=true "AIL framework termTop")
![Term-Plot](./doc/screenshots/terms-plot.png?raw=true "AIL framework termPlot")
![Term-tracker](./doc/screenshots/terms-tracker.png?raw=true "AIL framework termManager")
[AIL framework screencast](https://www.youtube.com/watch?v=1_ZrZkRKmNo)
@ -241,4 +235,3 @@ License
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
```

BIN
doc/screenshots/term-tracker.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 75 KiB

195
var/www/modules/terms/Flask_terms.py
View File

@ -150,105 +150,106 @@ def save_tag_to_auto_push(list_tag):
# ============ ROUTES ============
# TODO: remove + clean
@terms.route("/terms_plot_tool/")
@login_required
@login_read_only
def terms_plot_tool():
term = request.args.get('term')
if term is not None:
return render_template("terms_plot_tool.html", term=term)
else:
return render_template("terms_plot_tool.html", term="")
# @terms.route("/terms_plot_tool/")
# @login_required
# @login_read_only
# def terms_plot_tool():
# term = request.args.get('term')
# if term is not None:
# return render_template("terms_plot_tool.html", term=term)
# else:
# return render_template("terms_plot_tool.html", term="")
#
#
# @terms.route("/terms_plot_tool_data/")
# @login_required
# @login_read_only
# def terms_plot_tool_data():
# oneDay = 60*60*24
# range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
# range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
# range_start = calendar.timegm(range_start.timetuple())
# range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
# range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
# range_end = calendar.timegm(range_end.timetuple())
# term = request.args.get('term')
#
# per_paste = request.args.get('per_paste')
# if per_paste == "1" or per_paste is None:
# per_paste = "per_paste_"
# else:
# per_paste = ""
#
# if term is None:
# return "None"
#
# else:
# value_range = []
# for timestamp in range(range_start, range_end+oneDay, oneDay):
# value = r_serv_term.hget(per_paste+str(timestamp), term)
# curr_value_range = int(value) if value is not None else 0
# value_range.append([timestamp, curr_value_range])
# value_range.insert(0,term)
# return jsonify(value_range)
#
@terms.route("/terms_plot_tool_data/")
@login_required
@login_read_only
def terms_plot_tool_data():
oneDay = 60*60*24
range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
range_start = calendar.timegm(range_start.timetuple())
range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
range_end = calendar.timegm(range_end.timetuple())
term = request.args.get('term')
per_paste = request.args.get('per_paste')
if per_paste == "1" or per_paste is None:
per_paste = "per_paste_"
else:
per_paste = ""
if term is None:
return "None"
else:
value_range = []
for timestamp in range(range_start, range_end+oneDay, oneDay):
value = r_serv_term.hget(per_paste+str(timestamp), term)
curr_value_range = int(value) if value is not None else 0
value_range.append([timestamp, curr_value_range])
value_range.insert(0,term)
return jsonify(value_range)
@terms.route("/terms_plot_top/")
@login_required
@login_read_only
def terms_plot_top():
per_paste = request.args.get('per_paste')
per_paste = per_paste if per_paste is not None else 1
return render_template("terms_plot_top.html", per_paste=per_paste)
@terms.route("/terms_plot_top_data/")
@login_required
@login_read_only
def terms_plot_top_data():
oneDay = 60*60*24
today = datetime.datetime.now()
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
per_paste = request.args.get('per_paste')
if per_paste == "1" or per_paste is None:
per_paste = "per_paste_"
else:
per_paste = ""
set_day = per_paste + "TopTermFreq_set_day_" + str(today_timestamp)
set_week = per_paste + "TopTermFreq_set_week";
set_month = per_paste + "TopTermFreq_set_month";
the_set = per_paste + request.args.get('set')
num_day = int(request.args.get('num_day'))
if the_set is None:
return "None"
else:
to_return = []
if "TopTermFreq_set_day" in the_set:
the_set += "_" + str(today_timestamp)
for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
position = {}
position['day'] = r_serv_term.zrevrank(set_day, term)
position['day'] = position['day']+1 if position['day'] is not None else "<20"
position['week'] = r_serv_term.zrevrank(set_week, term)
position['week'] = position['week']+1 if position['week'] is not None else "<20"
position['month'] = r_serv_term.zrevrank(set_month, term)
position['month'] = position['month']+1 if position['month'] is not None else "<20"
value_range = []
for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
value = r_serv_term.hget(per_paste+str(timestamp), term)
curr_value_range = int(value) if value is not None else 0
value_range.append([timestamp, curr_value_range])
to_return.append([term, value_range, tot_value, position])
return jsonify(to_return)
# @terms.route("/terms_plot_top/"
# @login_required
# @login_read_only
# def terms_plot_top():
# per_paste = request.args.get('per_paste')
# per_paste = per_paste if per_paste is not None else 1
# return render_template("terms_plot_top.html", per_paste=per_paste)
#
#
# @terms.route("/terms_plot_top_data/")
# @login_required
# @login_read_only
# def terms_plot_top_data():
# oneDay = 60*60*24
# today = datetime.datetime.now()
# today = today.replace(hour=0, minute=0, second=0, microsecond=0)
# today_timestamp = calendar.timegm(today.timetuple())
#
# per_paste = request.args.get('per_paste')
# if per_paste == "1" or per_paste is None:
# per_paste = "per_paste_"
# else:
# per_paste = ""
#
# set_day = per_paste + "TopTermFreq_set_day_" + str(today_timestamp)
# set_week = per_paste + "TopTermFreq_set_week";
# set_month = per_paste + "TopTermFreq_set_month";
#
# the_set = per_paste + request.args.get('set')
# num_day = int(request.args.get('num_day'))
#
# if the_set is None:
# return "None"
# else:
# to_return = []
# if "TopTermFreq_set_day" in the_set:
# the_set += "_" + str(today_timestamp)
#
# for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
# position = {}
# position['day'] = r_serv_term.zrevrank(set_day, term)
# position['day'] = position['day']+1 if position['day'] is not None else "<20"
# position['week'] = r_serv_term.zrevrank(set_week, term)
# position['week'] = position['week']+1 if position['week'] is not None else "<20"
# position['month'] = r_serv_term.zrevrank(set_month, term)
# position['month'] = position['month']+1 if position['month'] is not None else "<20"
# value_range = []
# for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
# value = r_serv_term.hget(per_paste+str(timestamp), term)
# curr_value_range = int(value) if value is not None else 0
# value_range.append([timestamp, curr_value_range])
#
# to_return.append([term, value_range, tot_value, position])
#
# return jsonify(to_return)
@terms.route("/credentials_tracker/")

2
var/www/modules/terms/templates/header_terms.html
View File

@ -2,7 +2,5 @@
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
<li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
<li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
</ul>
</li>

1
var/www/modules/trendingcharts/templates/header_trendingcharts.html
View File

@ -1 +0,0 @@
<li id='page-trendingchart'><a href="{{ url_for('trendings.trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>