Added draft of terms_management + Added copyright in readme

pull/68/head
Mokaddem 2016-08-19 13:34:02 +02:00
parent 52b7124a6b
commit b78effe779
7 changed files with 514 additions and 29 deletions

View File

@ -104,6 +104,7 @@ LICENSE
Copyright (C) 2014-2016 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique)
Copyright (c) 2014-2016 Raphaël Vinot
Copyright (c) 2014-2016 Alexandre Dulaunoy
Copyright (c) 2016 Sami Mokaddem
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by

View File

@ -33,7 +33,7 @@ import calendar
from Helper import Process
# Config Variables
top_term_freq_max_set_cardinality = 50 # Max cardinality of the terms frequences set
top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
def getValueOverRange(word, startDate, num_day):
@ -76,29 +76,49 @@ if __name__ == "__main__":
message = p.get_from_set()
prec_filename = None
generate_new_graph = False
iii = 0
# Term Frequency
top_termFreq_setName_day = ["TopTermFreq_set_day", 1]
top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
while True:
if message is not None:
generate_new_graph = True
filename, word, score = message.split()
temp = filename.split('/')
date = temp[-4] + temp[-3] + temp[-2]
timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0))
# If set size is greater then the one authorized
# suppress smaller elements
for curr_set, curr_num_day in top_termFreq_set_array:
diffCard = server_term.scard(curr_set) - top_term_freq_max_set_cardinality
if diffCard > 0:
top_termFreq = server_term.smembers(curr_set)
sorted_top_termFreq_set = []
for word in top_termFreq:
word_value = getValueOverRange(word, timestamp, curr_num_day)
sorted_top_termFreq_set.append((word, word_value))
sorted_top_termFreq_set.sort(key=lambda tup: tup[1])
for i in range(0, diffCard):
print 'set oversized, dropping', sorted_top_termFreq_set[i][0]
server_term.srem(curr_set, sorted_top_termFreq_set[i][0])
#timer = time.clock()
low_word = word.lower()
#print 'wordlower', time.clock() - timer
r_serv1.hincrby(low_word, date, int(score))
# Term Frequency
top_termFreq_setName_day = ["TopTermFreq_set_day", 1]
top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0))
# Update redis
curr_word_value = int(server_term.hincrby(timestamp, low_word, int(score)))
# print '+----------------------------------------------------------------'
# Manage Top set
for curr_set, curr_num_day in top_termFreq_set_array:
@ -108,27 +128,23 @@ if __name__ == "__main__":
continue
else:
#timer = time.clock()
curr_word_value = getValueOverRange(low_word, timestamp, curr_num_day)
#print 'curr_range', time.clock() - timer
top_termFreq = server_term.smembers(curr_set)
sorted_top_termFreq_set = []
#timer = time.clock()
for word in top_termFreq:
word_value = getValueOverRange(word, timestamp, curr_num_day)
sorted_top_termFreq_set.append((word, word_value))
sorted_top_termFreq_set.sort(key=lambda tup: tup[1])
# if curr_num_day == 1:
# print sorted_top_termFreq_set
curr_word_value = getValueOverRange(low_word, timestamp, curr_num_day)
#print 'whole_range', time.clock() - timer
if curr_word_value > int(sorted_top_termFreq_set[0][1]):
print str(curr_num_day)+':', low_word, curr_word_value, '\t', sorted_top_termFreq_set[0][0], sorted_top_termFreq_set[0][1], '\t', curr_word_value > sorted_top_termFreq_set[0][1]
#print sorted_top_termFreq_set
server_term.srem(curr_set, sorted_top_termFreq_set[0][0])
server_term.sadd(curr_set, low_word)
if iii == 2:
iii-=1
else:
iii+=1
else:
if generate_new_graph:

View File

@ -136,7 +136,7 @@ function launching_scripts {
sleep 0.1
screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
#screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
sleep 0.1
@ -157,6 +157,13 @@ function launching_scripts {
screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "SentimentAnalyser" bash -c './SentimentAnalyser.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
}
#If no params, display the help

View File

@ -55,6 +55,11 @@ r_serv_sentiment = redis.StrictRedis(
port=cfg.getint("Redis_Level_DB_Sentiment", "port"),
db=cfg.getint("Redis_Level_DB_Sentiment", "db"))
r_serv_term = redis.StrictRedis(
host=cfg.get("Redis_Level_DB_TermFreq", "host"),
port=cfg.getint("Redis_Level_DB_TermFreq", "port"),
db=cfg.getint("Redis_Level_DB_TermFreq", "db"))
app = Flask(__name__, static_url_path='/static/')
@ -552,19 +557,71 @@ def sentiment_analysis_plot_tool_getdata():
return jsonify(to_return)
@app.route("/terms_management/")
def terms_management():
TrackedTermsSet_Name = "TrackedSetTermSet"
BlackListTermsSet_Name = "BlackListSetTermSet"
track_list = []
track_list_values = []
for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
track_list.append(tracked_term)
black_list = []
black_list_values = []
for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name):
black_list.append(blacked_term)
return render_template("terms_management.html", black_list=black_list, track_list=track_list)
@app.route("/terms_management_action/", methods=['GET'])
def terms_management_action():
TrackedTermsSet_Name = "TrackedSetTermSet"
BlackListTermsSet_Name = "BlackListSetTermSet"
section = request.args.get('section')
action = request.args.get('action')
term = request.args.get('term')
if action is None or term is None:
return "None"
else:
if section == "followTerm":
if action == "add":
r_serv_term.sadd(TrackedTermsSet_Name, term)
else:
r_serv_term.srem(TrackedTermsSet_Name, term)
elif section == "blacklistTerm":
if action == "add":
r_serv_term.sadd(BlackListTermsSet_Name, term)
else:
r_serv_term.srem(BlackListTermsSet_Name, term)
else:
return "None"
to_return = {}
to_return["section"] = section
to_return["action"] = action
to_return["term"] = term
return jsonify(to_return)
@app.route("/terms_plot_tool/")
def terms_plot_tool():
return render_template("terms_plot_tool.html")
@app.route("/test/") #completely shows the paste in a new tab
def test():
server = redis.StrictRedis(
host=cfg.get("Redis_Level_DB_TermFreq", "host"),
port=cfg.getint("Redis_Level_DB_TermFreq", "port"),
db=cfg.getint("Redis_Level_DB_TermFreq", "db"))
server = r_serv_term
array1 = []
for w in server.smembers('TopTermFreq_set_day'):
val = server.hget('1471478400', w)
val = server.hget('1471564800', w)
val = val if val is not None else 0
val2 = server.hget('1471392000', w)
val2 = server.hget('1471478400', w)
val2 = val2 if val2 is not None else 0
array1.append((w, (int(val), int(val2))))
@ -572,7 +629,7 @@ def test():
# for w in server.smembers('TopTermFreq_set_week'):
# array2.append((w, int(server.hget('1471478400', w))))
array1.sort(key=lambda tup: tup[1][0]+tup[1][1])
array1.sort(key=lambda tup: tup[1][0])
stri = "<h1> day </h1>"
for e in array1:
stri += "<p>"+ e[0] + "\t" + str(e[1]) +"</p>"

View File

@ -7,8 +7,15 @@
<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-heart"></i> Sentiment Analysis
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('sentiment_analysis_trending') }}">Trending</a></li>
<li><a href="{{ url_for('sentiment_analysis_plot_tool') }}">Plot Tool</a></li>
<li><a href="{{ url_for('sentiment_analysis_trending') }}"> Sentiment trending</a></li>
<li><a href="{{ url_for('sentiment_analysis_plot_tool') }}">Sentiment plot Tool</a></li>
</ul>
</li>
<li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-eye"></i> Terms frequency
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('terms_management') }}">Terms managements</a></li>
<li><a href="{{ url_for('terms_plot_tool') }}">Terms plot Tool</a></li>
</ul>
</li>
</ul>

View File

@ -0,0 +1,247 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.sparkLineStats ul {
padding-left:0;
list-style:none
}
.btn-link {
color: #000000
}
</style>
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Terms management interface</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-success">
<div class="panel-heading">
<strong>Manage tracked terms</strong>
</div>
<div class="panel-body">
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
<input id="followTermInput" class="form-control" placeholder="Term to track" type="text" style="max-width: 400px;">
<button class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable">
<thead>
<tr>
<th style="max-width: 800px;">Term name</th>
<th>Date</th>
<th>Day occurence</th>
<th>Week occurence</th>
<th>Month occurence</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% set i = 0 %}
{% for term in track_list %}
<tr>
<td>{{ term }}</td>
<td>{{ i }}</td>
<td>15</td>
<td>55</td>
<td>459</td>
<td><p style="margin: 0px;">
<button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="right" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
</p></td>
</tr>
{% set i = i + 1 %}
{% endfor %}
</tbody>
</table>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel OPTIONS -->
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-danger">
<div class="panel-heading">
<strong>Manage blacklisted terms</strong>
</div>
<div class="panel-body">
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye-slash "></span></span>
<input id="blacklistTermInput" class="form-control" placeholder="Term to track" type="text" style="max-width: 400px;">
<button class="btn btn-danger btn-interaction" style="margin-left: 10px;" data-section="blacklistTerm" data-action="add"> Black list a term</button>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable2">
<thead>
<tr>
<th style="max-width: 800px;">Term name</th>
<th>Date</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% set i = 0 %}
{% for term in black_list %}
<tr>
<td>{{ term }}</td>
<td>{{ i }}</td>
<td><p style="margin: 0px;">
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="right" title="Remove this term" data-content="{{ term }}" data-section="blacklistTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
</p></td>
</tr>
{% set i = i + 1 %}
{% endfor %}
</tbody>
</table>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<!-- import graph function -->
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
$('[data-toggle="tooltip"]').tooltip();
table_track = $('#myTable').DataTable();
table_black = $('#myTable2').DataTable();
perform_binding();
});
</script>
<script>
function perform_binding() {
$(".btn-interaction").unbind("click.interaction");
$(".btn-interaction").bind("click.interaction", perform_operation);
}
function perform_operation(){
var curr_section = $(this).attr('data-section');
var curr_action = $(this).attr('data-action');
if (curr_action == "add") {
var curr_term = $('#'+curr_section+'Input').val();
} else {
var curr_term = $(this).attr('data-content');
}
var data_to_send = { section: curr_section, action:curr_action, term: curr_term};
if (curr_term != "") {
console.log(data_to_send);
$.get("{{ url_for('terms_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
var json = data;
if(json.section == "followTerm") {
if(json.action == "add") {
// query data
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
table_track.row.add( [ json.term, 'now', 0, 0, 0, action_button ] ).draw( false );
perform_binding();
} else if (json.action == "delete") {
// Find indexes of row which have the term in the first column
var index = table_track.row().eq( 0 ).filter( function (rowIdx) {
return table_track.cell( rowIdx, 0 ).data() == json.term ? true : false;
} );
table_track.row().remove(index).draw( false );
}
} else if(json.section == "blacklistTerm"){
if(json.action == "add") {
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"blacklistTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
table_black.row.add( [ json.term, 'now', action_button ] ).draw( false );
perform_binding();
} else if (json.action == "delete") {
// Find indexes of row which have the term in the first column
var index = table_black.row().eq( 0 ).filter( function (rowIdx) {
return table_black.cell( rowIdx, 0 ).data() == json.term ? true : false;
} );
table_black.row().remove(index).draw( false );
}
}
}
});
}
}
</script>

View File

@ -0,0 +1,150 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-ui.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.sparkLineStats ul {
padding-left:0;
list-style:none
}
</style>
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-sentiment" >Terms plot tool</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Select options</strong>
</div>
<div class="panel-body">
<div class="row">
<!-- left column -->
<div class="col-lg-9">
<!-- providers charts -->
<div class="col-lg-6">
<div class="sparkLineStats">
<ul id="providerList1">
</ul>
</div>
</div>
<div class="col-lg-6">
<div class="sparkLineStats">
<ul id="providerList2">
</ul>
</div>
</div>
</div>
<!-- right column -->
<div class="col-lg-3">
<div aria-disabled="false" class="slider sliderRange sliderBlue ui-slider ui-slider-horizontal ui-widget ui-widget-content ui-corner-all" style="margin-bottom: 5px;"></div>
<strong>Date:</strong> <input type="text" id="amount" readonly style="border:0; color:#f6931f; font-weight:bold;">
</div>
</div>
<!-- /.row -->
<div class="row">
<button id="plot_btn" class="btn btn-info" style="margin-right: 8px; margin-left: 8px;float: right;">Plot!</button>
<label style="float: right;"><input id="checkbox_stacked" type="checkbox" checked=true> Stacked graph?</label>
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel PLOT -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph" style="height: 300px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<!-- import graph function -->
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
});
</script>
<script src="{{ url_for('static', filename='js/sentiment_plot.js') }}"></script>
</body>
</html>