CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

pull/63/head
Mokaddem 2016-07-21 14:04:18 +02:00
parent 4bc84a2580 ede46ec799
commit 37557da8a2
11 changed files with 133 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
.travis.yml
View File

@ -7,8 +7,6 @@ sudo: required
dist: trusty
group: edge
addons:
apt:
packages:
@ -27,6 +25,7 @@ addons:
# Leveldb
- libgmp-dev
- libev-dev
- cmake
env:
- AIL_HOME=$TRAVIS_BUILD_DIR AIL_BIN=$TRAVIS_BUILD_DIR/bin/ \
@ -38,14 +37,14 @@ env:
install:
- pip install -U pip
# DNS
- sudo apt-get install libadns1 libadns1-dev screen
- sudo apt-get install -y libadns1 libadns1-dev screen
# required for mathplotlib
- test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
- pip install distribute
# Redis
- test ! -d redis/ && git clone https://github.com/antirez/redis.git
- pushd redis
- git checkout 3.0
- git checkout 3.2
- make
- popd
# Redis leveldb
@ -55,6 +54,20 @@ install:
- git submodule update
- make
- popd
# Faup
- test ! -d faup && git clone https://github.com/stricaud/faup.git
- pushd faup/
- test ! -d build && mkdir build
- cd build
- cmake .. && make
- sudo make install
- echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf
- sudo ldconfig
- popd
# PyFaup
- pushd faup/src/lib/bindings/python/
- python setup.py install
- popd
# Set config
- cp bin/packages/config.cfg.sample bin/packages/config.cfg
- mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}

39
bin/WebStats.py
View File

@ -6,22 +6,21 @@
import time
import datetime
import re
import redis
import os
from packages import lib_words
from packages.Date import Date
from pubsublogger import publisher
from packages import Paste
from Helper import Process
from pyfaup.faup import Faup
# Config Var
threshold_need_to_look = 50
range_to_look = 10
threshold_to_plot = 1 #500%
threshold_to_plot = 1 # 500%
to_plot = set()
clean_frequency = 10 #minutes
clean_frequency = 10 # minutes
def analyse(server, field_name):
field = url_parsed[field_name]
@ -32,6 +31,7 @@ def analyse(server, field_name):
else:
server.hset(field, date, 1)
def analyse_and_progression(server, field_name):
field = url_parsed[field_name]
if field is not None:
@ -39,18 +39,19 @@ def analyse_and_progression(server, field_name):
if prev_score is not None:
print field + ' prev_score:' + prev_score
server.hset(field, date, int(prev_score) + 1)
if int(prev_score) + 1 > threshold_need_to_look: #threshold for false possitive
if int(prev_score) + 1 > threshold_need_to_look: # threshold for false possitive
if(check_for_progression(server, field, date)):
to_plot.add(field)
else:
server.hset(field, date, 1)
def check_for_progression(server, field, date):
previous_data = set()
tot_sum = 0
for i in range(0, range_to_look):
curr_value = server.hget(field, Date(date).substract_day(i))
if curr_value is None: #no further data
if curr_value is None: # no further data
break
else:
curr_value = int(curr_value)
@ -59,23 +60,13 @@ def check_for_progression(server, field, date):
if i == 0:
today_val = curr_value
print 'totsum='+str(tot_sum)
print 'div='+str(tot_sum/today_val)
if tot_sum/today_val >= threshold_to_plot:
print 'totsum=' + str(tot_sum)
print 'div=' + str(tot_sum / today_val)
if tot_sum / today_val >= threshold_to_plot:
return True
else:
return False
def clean_to_plot():
temp_to_plot = set()
curr_date = datetime.date.today()
date = Date(str(curr_date.year)+str(curr_date.month)+str(curr_date.day))
for elem in to_plot:
if(check_for_progression(field, date)):
temp_to_plot.add(elem)
to_plot = temp_to_plot
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
@ -118,7 +109,6 @@ if __name__ == '__main__':
csv_path_domain = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "domainstrending_csv"))
faup = Faup()
generate_new_graph = False
# Endless loop getting messages from the input queue
@ -143,8 +133,7 @@ if __name__ == '__main__':
month)
lib_words.create_curve_with_list(r_serv2, csv_path_domain,
to_plot, year,
month)
to_plot, year, month)
print 'end building'
publisher.debug("{} queue is empty, waiting".format(config_section))
@ -159,6 +148,6 @@ if __name__ == '__main__':
faup.decode(url)
url_parsed = faup.get()
analyse(r_serv1, 'scheme') #Scheme analysis
analyse(r_serv1, 'tld') #Tld analysis
analyse_and_progression(r_serv2, 'domain') #Domain analysis
analyse(r_serv1, 'scheme') # Scheme analysis
analyse(r_serv1, 'tld') # Tld analysis
analyse_and_progression(r_serv2, 'domain') # Domain analysis

3
bin/launch_scripts.sh
View File

@ -35,4 +35,5 @@ sleep 0.1
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'

30
bin/packages/config.cfg.sample
View File

@ -1,9 +1,30 @@
[Directories]
bloomfilters = Blooms
pastes = PASTES
wordtrending_csv = var/www/static/csv/wordstrendingdata
wordsfile = files/wordfile
protocolstrending_csv = var/www/static/csv/protocolstrendingdata
protocolsfile = files/protocolsfile
tldstrending_csv = var/www/static/csv/tldstrendingdata
tldsfile = AILENV/faup/src/data/mozilla.tlds
domainstrending_csv = var/www/static/csv/domainstrendingdata
##### Flask #####
[Flask]
#Number of minutes displayed for the number of processed pastes.
minute_processed_paste = 10
#Maximum number of character to display in the toolip
max_preview_char = 250
#Maximum number of character to display in the modal
max_preview_modal = 800
#Default number of header to display in trending graphs
default_display = 10
##### Redis #####
[Redis_Cache]
host = localhost
@ -28,12 +49,17 @@ db = 1
##### LevelDB #####
[Redis_Level_DB]
host = localhost
port = 2013
port = 2016
db = 0
[Redis_Level_DB_Domain]
host = localhost
port = 2016
db = 3
[Redis_Level_DB_Hashs]
host = localhost
port = 2013
port = 2016
db = 1
[Url]

21
installing_deps.sh
View File

@ -6,7 +6,7 @@ set -x
sudo apt-get update
sudo apt-get install python-pip python-virtualenv python-dev libfreetype6-dev \
screen g++ python-tk unzip libsnappy-dev
screen g++ python-tk unzip libsnappy-dev cmake
#Needed for bloom filters
sudo apt-get install libssl-dev libfreetype6-dev python-numpy
@ -24,10 +24,21 @@ sudo easy_install -U distribute
# REDIS #
test ! -d redis/ && git clone https://github.com/antirez/redis.git
pushd redis/
git checkout 3.0
git checkout 3.2
make
popd
# Faup
test ! -d faup && git clone https://github.com/stricaud/faup.git
pushd faup/
test ! -d build && mkdir build
cd build
cmake .. && make
sudo make install
echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf
sudo ldconfig
popd
# REDIS LEVEL DB #
test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git
pushd redis-leveldb/
@ -56,5 +67,11 @@ mkdir -p $AIL_HOME/LEVEL_DB_DATA/2016
pip install -U pip
pip install -r pip_packages_requirement.txt
# Pyfaup
pushd faup/src/lib/bindings/python/
python setup.py install
popd
# Download the necessary NLTK corpora
HOME=$(pwd) python -m textblob.download_corpora

45
var/www/static/js/indexjavascript.js
View File

@ -1,52 +1,11 @@
// Plot and update the number of processed pastes
$(function() {
var data = [];
var totalPoints = 60*10; //60s*10m
var default_minute = (typeof window.default_minute !== "undefined") ? parseInt(window.default_minute) : 10;
var totalPoints = 60*parseInt(default_minute); //60s*minute
var curr_max = 0;
function getData() {
if (data.length > 0){
var data_old = data[0];
data = data.slice(1);
curr_max = curr_max == data_old ? Math.max.apply(null, data) : curr_max;
}
while (data.length < totalPoints) {
var y = (typeof window.paste_num_tabvar !== "undefined") ? parseInt(window.paste_num_tabvar) : 0;
curr_max = y > curr_max ? y : curr_max;
data.push(y);
}
// Zip the generated y values with the x values
var res = [];
for (var i = 0; i < data.length; ++i) {
res.push([i, data[i]])
}
return res;
}
var updateInterval = 1000;
var options = {
series: { shadowSize: 1 },
lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }},
yaxis: { min: 0, max: 40 },
colors: ["#a971ff"],
grid: {
tickColor: "#dddddd",
borderWidth: 0
},
};
var plot = $.plot("#realtimechart", [ getData() ], options);
function update() {
plot.setData([getData()]);
plot.getOptions().yaxes[0].max = curr_max;
plot.setupGrid();
plot.draw();
setTimeout(update, updateInterval);
}
update();
});
function initfunc( csvay, scroot) {
window.csv = csvay;
window.scroot = scroot;

1
var/www/templates/Queue_live_Monitoring.html
View File

@ -1,7 +1,6 @@
<html>
<head>
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
</head>
<body>
<title>WordsTrend</title>

1
var/www/templates/Trending.html
View File

@ -14,7 +14,6 @@
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script>
var default_display = {{ default_display }};

3
var/www/templates/index.html
View File

@ -14,11 +14,10 @@
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.tagcanvas.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script>
window.default_minute = {{ default_minute }};
function update_values() {
$SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
$.getJSON($SCRIPT_ROOT+"/_stuff",

1
var/www/templates/search.html
View File

@ -15,7 +15,6 @@
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>

48
var/www/update_thirdparty.sh
View File

@ -4,26 +4,30 @@ set -e
wget http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
SBADMIN_VERSION=2
filename="sb-admin-${SBADMIN_VERSION}"
SBADMIN_VERSION='1.0.4'
rm -rf temp
mkdir temp
wget https://github.com/IronSummitMedia/startbootstrap-sb-admin-2/archive/v1.0.2.zip -O temp/${filename}".zip"
unzip temp/${filename}".zip" -d temp/
mv temp/startbootstrap-sb-admin-2-1.0.2 temp/sb-admin-2
wget https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip
unzip temp/${SBADMIN_VERSION}.zip -d temp/
mv temp/startbootstrap-sb-admin-${SBADMIN_VERSION} temp/sb-admin-2
JQVERSION="1.11.1"
rm -rf ./static/js/plugins
mv temp/sb-admin-2/js/* ./static/js/
rm -rf ./static/fonts/ ./static/font-awesome/
mv temp/sb-admin-2/fonts/ ./static/
mv temp/sb-admin-2/font-awesome/ ./static/
rm -rf ./static/css/plugins/
mv temp/sb-admin-2/css/* ./static/css/
rm -rf temp
JQVERSION="1.12.4"
wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
#wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
#Here to fix an error about an hard dependency in a obscur script of bootstrap..
wget http://code.jquery.com/jquery-1.4.2.js -O ./static/js/jquery-1.4.2.js
wget http://www.goat1000.com/jquery.tagcanvas.js?2.5 -O ./static/js/jquery.tagcanvas.js
#Ressources for dataTable
wget https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js
wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css
@ -33,21 +37,7 @@ wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTabl
wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.js -O ./static/js/jquery.flot.js
wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js
rm -rf ./static/js/plugins
mv temp/${filename}/js/* ./static/js/
rm -rf ./static/fonts/ ./static/font-awesome-4.1.0/
mv temp/${filename}/fonts/ ./static/
mv temp/${filename}/font-awesome/ ./static/
rm -rf ./static/css/plugins/
mv temp/${filename}/css/* ./static/css/
rm -rf temp/
mkdir -p ./static/image
cd static/image
pushd static/image
wget https://www.circl.lu/assets/images/logos/AIL.png -O AIL.png
cd ../..
popd