Merge remote-tracking branch 'upstream/master'

pull/63/head
Mokaddem 2016-07-21 14:04:18 +02:00
commit 37557da8a2
11 changed files with 133 additions and 142 deletions

View File

@ -7,8 +7,6 @@ sudo: required
dist: trusty
group: edge
addons:
apt:
packages:
@ -27,6 +25,7 @@ addons:
# Leveldb
- libgmp-dev
- libev-dev
- cmake
env:
- AIL_HOME=$TRAVIS_BUILD_DIR AIL_BIN=$TRAVIS_BUILD_DIR/bin/ \
@ -38,14 +37,14 @@ env:
install:
- pip install -U pip
# DNS
- sudo apt-get install libadns1 libadns1-dev screen
- sudo apt-get install -y libadns1 libadns1-dev screen
# required for mathplotlib
- test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
- pip install distribute
# Redis
- test ! -d redis/ && git clone https://github.com/antirez/redis.git
- pushd redis
- git checkout 3.0
- git checkout 3.2
- make
- popd
# Redis leveldb
@ -55,6 +54,20 @@ install:
- git submodule update
- make
- popd
# Faup
- test ! -d faup && git clone https://github.com/stricaud/faup.git
- pushd faup/
- test ! -d build && mkdir build
- cd build
- cmake .. && make
- sudo make install
- echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf
- sudo ldconfig
- popd
# PyFaup
- pushd faup/src/lib/bindings/python/
- python setup.py install
- popd
# Set config
- cp bin/packages/config.cfg.sample bin/packages/config.cfg
- mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}

View File

@ -6,22 +6,21 @@
import time
import datetime
import re
import redis
import os
from packages import lib_words
from packages.Date import Date
from pubsublogger import publisher
from packages import Paste
from Helper import Process
from pyfaup.faup import Faup
# Config Var
threshold_need_to_look = 50
range_to_look = 10
threshold_to_plot = 1 #500%
to_plot = set()
clean_frequency = 10 #minutes
threshold_need_to_look = 50
range_to_look = 10
threshold_to_plot = 1 # 500%
to_plot = set()
clean_frequency = 10 # minutes
def analyse(server, field_name):
field = url_parsed[field_name]
@ -32,6 +31,7 @@ def analyse(server, field_name):
else:
server.hset(field, date, 1)
def analyse_and_progression(server, field_name):
field = url_parsed[field_name]
if field is not None:
@ -39,43 +39,34 @@ def analyse_and_progression(server, field_name):
if prev_score is not None:
print field + ' prev_score:' + prev_score
server.hset(field, date, int(prev_score) + 1)
if int(prev_score) + 1 > threshold_need_to_look: #threshold for false possitive
if int(prev_score) + 1 > threshold_need_to_look: # threshold for false possitive
if(check_for_progression(server, field, date)):
to_plot.add(field)
else:
server.hset(field, date, 1)
def check_for_progression(server, field, date):
previous_data = set()
tot_sum = 0
for i in range(0, range_to_look):
curr_value = server.hget(field, Date(date).substract_day(i))
if curr_value is None: #no further data
if curr_value is None: # no further data
break
else:
curr_value = int(curr_value)
previous_data.add(curr_value)
tot_sum += curr_value
tot_sum += curr_value
if i == 0:
today_val = curr_value
print 'totsum='+str(tot_sum)
print 'div='+str(tot_sum/today_val)
if tot_sum/today_val >= threshold_to_plot:
print 'totsum=' + str(tot_sum)
print 'div=' + str(tot_sum / today_val)
if tot_sum / today_val >= threshold_to_plot:
return True
else:
return False
def clean_to_plot():
temp_to_plot = set()
curr_date = datetime.date.today()
date = Date(str(curr_date.year)+str(curr_date.month)+str(curr_date.day))
for elem in to_plot:
if(check_for_progression(field, date)):
temp_to_plot.add(elem)
to_plot = temp_to_plot
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
@ -98,7 +89,7 @@ if __name__ == '__main__':
host=p.config.get("Redis_Level_DB", "host"),
port=p.config.get("Redis_Level_DB", "port"),
db=p.config.get("Redis_Level_DB", "db"))
r_serv2 = redis.StrictRedis(
host=p.config.get("Redis_Level_DB_Domain", "host"),
port=p.config.get("Redis_Level_DB_Domain", "port"),
@ -106,18 +97,17 @@ if __name__ == '__main__':
# FILE CURVE SECTION #
csv_path_proto = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "protocolstrending_csv"))
p.config.get("Directories", "protocolstrending_csv"))
protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "protocolsfile"))
p.config.get("Directories", "protocolsfile"))
csv_path_tld = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "tldstrending_csv"))
p.config.get("Directories", "tldstrending_csv"))
tldsfile_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "tldsfile"))
csv_path_domain = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "domainstrending_csv"))
p.config.get("Directories", "domainstrending_csv"))
faup = Faup()
generate_new_graph = False
@ -125,7 +115,7 @@ if __name__ == '__main__':
while True:
# Get one message from the input queue
message = p.get_from_set()
if message is None:
if generate_new_graph:
generate_new_graph = False
@ -143,8 +133,7 @@ if __name__ == '__main__':
month)
lib_words.create_curve_with_list(r_serv2, csv_path_domain,
to_plot, year,
month)
to_plot, year, month)
print 'end building'
publisher.debug("{} queue is empty, waiting".format(config_section))
@ -152,13 +141,13 @@ if __name__ == '__main__':
time.sleep(5)
continue
else:
else:
generate_new_graph = True
# Do something with the message from the queue
url, date = message.split()
faup.decode(url)
url_parsed = faup.get()
analyse(r_serv1, 'scheme') #Scheme analysis
analyse(r_serv1, 'tld') #Tld analysis
analyse_and_progression(r_serv2, 'domain') #Domain analysis
analyse(r_serv1, 'scheme') # Scheme analysis
analyse(r_serv1, 'tld') # Tld analysis
analyse_and_progression(r_serv2, 'domain') # Domain analysis

View File

@ -35,4 +35,5 @@ sleep 0.1
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'

View File

@ -1,9 +1,30 @@
[Directories]
bloomfilters = Blooms
pastes = PASTES
wordtrending_csv = var/www/static/csv/wordstrendingdata
wordsfile = files/wordfile
protocolstrending_csv = var/www/static/csv/protocolstrendingdata
protocolsfile = files/protocolsfile
tldstrending_csv = var/www/static/csv/tldstrendingdata
tldsfile = AILENV/faup/src/data/mozilla.tlds
domainstrending_csv = var/www/static/csv/domainstrendingdata
##### Flask #####
[Flask]
#Number of minutes displayed for the number of processed pastes.
minute_processed_paste = 10
#Maximum number of character to display in the toolip
max_preview_char = 250
#Maximum number of character to display in the modal
max_preview_modal = 800
#Default number of header to display in trending graphs
default_display = 10
##### Redis #####
[Redis_Cache]
host = localhost
@ -28,12 +49,17 @@ db = 1
##### LevelDB #####
[Redis_Level_DB]
host = localhost
port = 2013
port = 2016
db = 0
[Redis_Level_DB_Domain]
host = localhost
port = 2016
db = 3
[Redis_Level_DB_Hashs]
host = localhost
port = 2013
port = 2016
db = 1
[Url]

View File

@ -6,7 +6,7 @@ set -x
sudo apt-get update
sudo apt-get install python-pip python-virtualenv python-dev libfreetype6-dev \
screen g++ python-tk unzip libsnappy-dev
screen g++ python-tk unzip libsnappy-dev cmake
#Needed for bloom filters
sudo apt-get install libssl-dev libfreetype6-dev python-numpy
@ -24,10 +24,21 @@ sudo easy_install -U distribute
# REDIS #
test ! -d redis/ && git clone https://github.com/antirez/redis.git
pushd redis/
git checkout 3.0
git checkout 3.2
make
popd
# Faup
test ! -d faup && git clone https://github.com/stricaud/faup.git
pushd faup/
test ! -d build && mkdir build
cd build
cmake .. && make
sudo make install
echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf
sudo ldconfig
popd
# REDIS LEVEL DB #
test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git
pushd redis-leveldb/
@ -56,5 +67,11 @@ mkdir -p $AIL_HOME/LEVEL_DB_DATA/2016
pip install -U pip
pip install -r pip_packages_requirement.txt
# Pyfaup
pushd faup/src/lib/bindings/python/
python setup.py install
popd
# Download the necessary NLTK corpora
HOME=$(pwd) python -m textblob.download_corpora

View File

@ -1,52 +1,11 @@
// Plot and update the number of processed pastes
$(function() {
var data = [];
var totalPoints = 60*10; //60s*10m
var default_minute = (typeof window.default_minute !== "undefined") ? parseInt(window.default_minute) : 10;
var totalPoints = 60*parseInt(default_minute); //60s*minute
var curr_max = 0;
function getData() {
if (data.length > 0){
var data_old = data[0];
data = data.slice(1);
curr_max = curr_max == data_old ? Math.max.apply(null, data) : curr_max;
}
while (data.length < totalPoints) {
var y = (typeof window.paste_num_tabvar !== "undefined") ? parseInt(window.paste_num_tabvar) : 0;
curr_max = y > curr_max ? y : curr_max;
data.push(y);
}
// Zip the generated y values with the x values
var res = [];
for (var i = 0; i < data.length; ++i) {
res.push([i, data[i]])
}
return res;
}
var updateInterval = 1000;
var options = {
series: { shadowSize: 1 },
lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }},
yaxis: { min: 0, max: 40 },
colors: ["#a971ff"],
grid: {
tickColor: "#dddddd",
borderWidth: 0
},
};
var plot = $.plot("#realtimechart", [ getData() ], options);
function update() {
plot.setData([getData()]);
plot.getOptions().yaxes[0].max = curr_max;
plot.setupGrid();
plot.draw();
setTimeout(update, updateInterval);
}
update();
});
function initfunc( csvay, scroot) {
window.csv = csvay;
window.scroot = scroot;

View File

@ -1,7 +1,6 @@
<html>
<head>
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
</head>
<body>
<title>WordsTrend</title>

View File

@ -14,9 +14,8 @@
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script>
<script>
var default_display = {{ default_display }};
var current_displayed_graph;
</script>
@ -56,7 +55,7 @@
</div>
<!-- /.row -->
<div class="row">
<!-- /.nav-tabs -->
<ul class="nav nav-tabs">
<li class="active"><a data-toggle="tab" href="#tld-tab" data-pannel="TldTrending" data-path="../static//csv/tldstrendingdata.csv">Top level domains</a></li>
@ -65,8 +64,8 @@
<li><a data-toggle="tab" href="#protocol-tab" data-pannel="ProtocolTrending" data-path="../static//csv/protocolstrendingdata.csv">Protocols</a></li>
</ul>
</br>
<div class="tab-content">
<div class="tab-content">
<div class="col-lg-12 tab-pane fade in active" id="tld-tab" >
{% include 'trending_graphs/Tldstrending.html' %}
</div>
@ -75,7 +74,7 @@
</div>
<div class="col-lg-12 tab-pane fade" id="words-tab">
{% include 'trending_graphs/Wordstrending.html' %}
</div>
</div>
<div class="col-lg-12 tab-pane fade" id="protocol-tab">
{% include 'trending_graphs/Protocolstrending.html' %}
</div>
@ -98,27 +97,27 @@
current_displayed_graph = new Graph(pannel, path, header_size);
setTimeout(function() { current_displayed_graph.set_Visibility(default_display)}, 300);
}, 'text');
}
// When a pannel is shown, create_and_plot.
$('.nav-tabs a').on('shown.bs.tab', function(event){
create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path'));
create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path'));
});
</script>
<script>
$(document).ready(function(){
// Create the graph when the page has just loaded
create_and_plot("TldTrending", '../static//csv/tldstrendingdata.csv')
create_and_plot("TldTrending", '../static//csv/tldstrendingdata.csv')
});
// Used when we modify the number of displayed curves
function take_top(new_display){
current_displayed_graph.set_Visibility_andHide(new_display, default_display);
default_display = new_display;
}
</script>
</div>

View File

@ -14,11 +14,10 @@
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.tagcanvas.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script>
window.default_minute = {{ default_minute }};
function update_values() {
$SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
$.getJSON($SCRIPT_ROOT+"/_stuff",
@ -108,7 +107,7 @@
<input id="checkbox_log_info" type="checkbox" value="info" checked="true"> INFO
<input id="checkbox_log_warning" type="checkbox" value="warning" checked="true"> WARNING
<input id="checkbox_log_critical" type="checkbox" value="critical" checked="true"> CRITICAL
</div>
</div>
<div class="panel-body">

View File

@ -15,12 +15,11 @@
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<!-- Custom style -->
<style>
<style>
.tooltip-inner {
text-align: left;
height: 200%;
@ -93,7 +92,7 @@
<div class="panel-heading">
<i class="glyphicon glyphicon-search"></i> {{ r|length }} Results for "<strong>{{ query }}</strong>"
<div class="pull-right">
</div>
</div>
<!-- /.panel-heading -->
@ -176,7 +175,7 @@
});
} else {
update_preview();
}
}
});
});
});
@ -191,7 +190,7 @@
complete_paste = null;
start_index = 0;
});
// Update the paste preview in the modal
function update_preview() {
if (start_index + char_to_display > complete_paste.length-1){ // end of paste reached
@ -203,13 +202,13 @@
if (final_index != start_index){ // still have data to display
$("#mymodalbody").find("#paste-holder").append(complete_paste.substring(start_index+1, final_index+1)); // Append the new content
start_index = final_index;
start_index = final_index;
if (flag_stop)
nothing_to_display();
} else {
nothing_to_display();
nothing_to_display();
}
}
}
// Update the modal when there is no more data
function nothing_to_display() {
var new_content = $(alert_message).hide();

View File

@ -4,26 +4,30 @@ set -e
wget http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
SBADMIN_VERSION=2
filename="sb-admin-${SBADMIN_VERSION}"
SBADMIN_VERSION='1.0.4'
rm -rf temp
mkdir temp
wget https://github.com/IronSummitMedia/startbootstrap-sb-admin-2/archive/v1.0.2.zip -O temp/${filename}".zip"
unzip temp/${filename}".zip" -d temp/
mv temp/startbootstrap-sb-admin-2-1.0.2 temp/sb-admin-2
wget https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip
unzip temp/${SBADMIN_VERSION}.zip -d temp/
mv temp/startbootstrap-sb-admin-${SBADMIN_VERSION} temp/sb-admin-2
JQVERSION="1.11.1"
rm -rf ./static/js/plugins
mv temp/sb-admin-2/js/* ./static/js/
rm -rf ./static/fonts/ ./static/font-awesome/
mv temp/sb-admin-2/fonts/ ./static/
mv temp/sb-admin-2/font-awesome/ ./static/
rm -rf ./static/css/plugins/
mv temp/sb-admin-2/css/* ./static/css/
rm -rf temp
JQVERSION="1.12.4"
wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
#wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
#Here to fix an error about an hard dependency in a obscur script of bootstrap..
wget http://code.jquery.com/jquery-1.4.2.js -O ./static/js/jquery-1.4.2.js
wget http://www.goat1000.com/jquery.tagcanvas.js?2.5 -O ./static/js/jquery.tagcanvas.js
#Ressources for dataTable
wget https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js
wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css
@ -33,21 +37,7 @@ wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTabl
wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.js -O ./static/js/jquery.flot.js
wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js
rm -rf ./static/js/plugins
mv temp/${filename}/js/* ./static/js/
rm -rf ./static/fonts/ ./static/font-awesome-4.1.0/
mv temp/${filename}/fonts/ ./static/
mv temp/${filename}/font-awesome/ ./static/
rm -rf ./static/css/plugins/
mv temp/${filename}/css/* ./static/css/
rm -rf temp/
mkdir -p ./static/image
cd static/image
pushd static/image
wget https://www.circl.lu/assets/images/logos/AIL.png -O AIL.png
cd ../..
popd