diff --git a/.travis.yml b/.travis.yml index e0e50939..1ce79b41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,6 @@ sudo: required dist: trusty -group: edge - addons: apt: packages: @@ -27,6 +25,7 @@ addons: # Leveldb - libgmp-dev - libev-dev + - cmake env: - AIL_HOME=$TRAVIS_BUILD_DIR AIL_BIN=$TRAVIS_BUILD_DIR/bin/ \ @@ -38,14 +37,14 @@ env: install: - pip install -U pip # DNS - - sudo apt-get install libadns1 libadns1-dev screen + - sudo apt-get install -y libadns1 libadns1-dev screen # required for mathplotlib - test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/ - pip install distribute # Redis - test ! -d redis/ && git clone https://github.com/antirez/redis.git - pushd redis - - git checkout 3.0 + - git checkout 3.2 - make - popd # Redis leveldb @@ -55,6 +54,20 @@ install: - git submodule update - make - popd + # Faup + - test ! -d faup && git clone https://github.com/stricaud/faup.git + - pushd faup/ + - test ! -d build && mkdir build + - cd build + - cmake .. && make + - sudo make install + - echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf + - sudo ldconfig + - popd + # PyFaup + - pushd faup/src/lib/bindings/python/ + - python setup.py install + - popd # Set config - cp bin/packages/config.cfg.sample bin/packages/config.cfg - mkdir -p $AIL_HOME/{PASTES,Blooms,dumps} diff --git a/bin/WebStats.py b/bin/WebStats.py index 71bbd944..5573b8fd 100755 --- a/bin/WebStats.py +++ b/bin/WebStats.py @@ -6,22 +6,21 @@ import time import datetime -import re import redis import os from packages import lib_words from packages.Date import Date from pubsublogger import publisher -from packages import Paste from Helper import Process from pyfaup.faup import Faup # Config Var -threshold_need_to_look = 50 -range_to_look = 10 -threshold_to_plot = 1 #500% -to_plot = set() -clean_frequency = 10 #minutes +threshold_need_to_look = 50 +range_to_look = 10 +threshold_to_plot = 1 # 500% +to_plot = set() +clean_frequency = 10 # minutes + def analyse(server, field_name): field = url_parsed[field_name] @@ -32,6 +31,7 @@ def analyse(server, field_name): else: server.hset(field, date, 1) + def analyse_and_progression(server, field_name): field = url_parsed[field_name] if field is not None: @@ -39,43 +39,34 @@ def analyse_and_progression(server, field_name): if prev_score is not None: print field + ' prev_score:' + prev_score server.hset(field, date, int(prev_score) + 1) - if int(prev_score) + 1 > threshold_need_to_look: #threshold for false possitive + if int(prev_score) + 1 > threshold_need_to_look: # threshold for false possitive if(check_for_progression(server, field, date)): to_plot.add(field) else: server.hset(field, date, 1) + def check_for_progression(server, field, date): previous_data = set() tot_sum = 0 for i in range(0, range_to_look): curr_value = server.hget(field, Date(date).substract_day(i)) - if curr_value is None: #no further data + if curr_value is None: # no further data break else: curr_value = int(curr_value) previous_data.add(curr_value) - tot_sum += curr_value + tot_sum += curr_value if i == 0: today_val = curr_value - - print 'totsum='+str(tot_sum) - print 'div='+str(tot_sum/today_val) - if tot_sum/today_val >= threshold_to_plot: + print 'totsum=' + str(tot_sum) + print 'div=' + str(tot_sum / today_val) + if tot_sum / today_val >= threshold_to_plot: return True else: return False -def clean_to_plot(): - temp_to_plot = set() - curr_date = datetime.date.today() - date = Date(str(curr_date.year)+str(curr_date.month)+str(curr_date.day)) - - for elem in to_plot: - if(check_for_progression(field, date)): - temp_to_plot.add(elem) - to_plot = temp_to_plot if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) @@ -98,7 +89,7 @@ if __name__ == '__main__': host=p.config.get("Redis_Level_DB", "host"), port=p.config.get("Redis_Level_DB", "port"), db=p.config.get("Redis_Level_DB", "db")) - + r_serv2 = redis.StrictRedis( host=p.config.get("Redis_Level_DB_Domain", "host"), port=p.config.get("Redis_Level_DB_Domain", "port"), @@ -106,18 +97,17 @@ if __name__ == '__main__': # FILE CURVE SECTION # csv_path_proto = os.path.join(os.environ['AIL_HOME'], - p.config.get("Directories", "protocolstrending_csv")) + p.config.get("Directories", "protocolstrending_csv")) protocolsfile_path = os.path.join(os.environ['AIL_HOME'], - p.config.get("Directories", "protocolsfile")) - + p.config.get("Directories", "protocolsfile")) + csv_path_tld = os.path.join(os.environ['AIL_HOME'], - p.config.get("Directories", "tldstrending_csv")) + p.config.get("Directories", "tldstrending_csv")) tldsfile_path = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "tldsfile")) csv_path_domain = os.path.join(os.environ['AIL_HOME'], - p.config.get("Directories", "domainstrending_csv")) - + p.config.get("Directories", "domainstrending_csv")) faup = Faup() generate_new_graph = False @@ -125,7 +115,7 @@ if __name__ == '__main__': while True: # Get one message from the input queue message = p.get_from_set() - + if message is None: if generate_new_graph: generate_new_graph = False @@ -143,8 +133,7 @@ if __name__ == '__main__': month) lib_words.create_curve_with_list(r_serv2, csv_path_domain, - to_plot, year, - month) + to_plot, year, month) print 'end building' publisher.debug("{} queue is empty, waiting".format(config_section)) @@ -152,13 +141,13 @@ if __name__ == '__main__': time.sleep(5) continue - else: + else: generate_new_graph = True # Do something with the message from the queue url, date = message.split() faup.decode(url) url_parsed = faup.get() - - analyse(r_serv1, 'scheme') #Scheme analysis - analyse(r_serv1, 'tld') #Tld analysis - analyse_and_progression(r_serv2, 'domain') #Domain analysis + + analyse(r_serv1, 'scheme') # Scheme analysis + analyse(r_serv1, 'tld') # Tld analysis + analyse_and_progression(r_serv2, 'domain') # Domain analysis diff --git a/bin/launch_scripts.sh b/bin/launch_scripts.sh index dc1bfa86..c7f4228f 100755 --- a/bin/launch_scripts.sh +++ b/bin/launch_scripts.sh @@ -35,4 +35,5 @@ sleep 0.1 screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' - +sleep 0.1 +screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 6d07707c..221cadde 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -1,9 +1,30 @@ [Directories] bloomfilters = Blooms pastes = PASTES + wordtrending_csv = var/www/static/csv/wordstrendingdata wordsfile = files/wordfile +protocolstrending_csv = var/www/static/csv/protocolstrendingdata +protocolsfile = files/protocolsfile + +tldstrending_csv = var/www/static/csv/tldstrendingdata +tldsfile = AILENV/faup/src/data/mozilla.tlds + +domainstrending_csv = var/www/static/csv/domainstrendingdata + +##### Flask ##### +[Flask] +#Number of minutes displayed for the number of processed pastes. +minute_processed_paste = 10 + +#Maximum number of character to display in the toolip +max_preview_char = 250 +#Maximum number of character to display in the modal +max_preview_modal = 800 +#Default number of header to display in trending graphs +default_display = 10 + ##### Redis ##### [Redis_Cache] host = localhost @@ -28,12 +49,17 @@ db = 1 ##### LevelDB ##### [Redis_Level_DB] host = localhost -port = 2013 +port = 2016 db = 0 +[Redis_Level_DB_Domain] +host = localhost +port = 2016 +db = 3 + [Redis_Level_DB_Hashs] host = localhost -port = 2013 +port = 2016 db = 1 [Url] diff --git a/installing_deps.sh b/installing_deps.sh index aa84235f..ae4f3fc8 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -6,7 +6,7 @@ set -x sudo apt-get update sudo apt-get install python-pip python-virtualenv python-dev libfreetype6-dev \ - screen g++ python-tk unzip libsnappy-dev + screen g++ python-tk unzip libsnappy-dev cmake #Needed for bloom filters sudo apt-get install libssl-dev libfreetype6-dev python-numpy @@ -24,10 +24,21 @@ sudo easy_install -U distribute # REDIS # test ! -d redis/ && git clone https://github.com/antirez/redis.git pushd redis/ -git checkout 3.0 +git checkout 3.2 make popd +# Faup +test ! -d faup && git clone https://github.com/stricaud/faup.git +pushd faup/ +test ! -d build && mkdir build +cd build +cmake .. && make +sudo make install +echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf +sudo ldconfig +popd + # REDIS LEVEL DB # test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git pushd redis-leveldb/ @@ -56,5 +67,11 @@ mkdir -p $AIL_HOME/LEVEL_DB_DATA/2016 pip install -U pip pip install -r pip_packages_requirement.txt +# Pyfaup +pushd faup/src/lib/bindings/python/ +python setup.py install +popd + + # Download the necessary NLTK corpora HOME=$(pwd) python -m textblob.download_corpora diff --git a/var/www/static/js/indexjavascript.js b/var/www/static/js/indexjavascript.js index 628ffe86..bd73522a 100644 --- a/var/www/static/js/indexjavascript.js +++ b/var/www/static/js/indexjavascript.js @@ -1,52 +1,11 @@ // Plot and update the number of processed pastes $(function() { var data = []; - var totalPoints = 60*10; //60s*10m + var default_minute = (typeof window.default_minute !== "undefined") ? parseInt(window.default_minute) : 10; + var totalPoints = 60*parseInt(default_minute); //60s*minute var curr_max = 0; function getData() { - if (data.length > 0){ - var data_old = data[0]; - data = data.slice(1); - curr_max = curr_max == data_old ? Math.max.apply(null, data) : curr_max; - } - - while (data.length < totalPoints) { - var y = (typeof window.paste_num_tabvar !== "undefined") ? parseInt(window.paste_num_tabvar) : 0; - curr_max = y > curr_max ? y : curr_max; - data.push(y); - } - // Zip the generated y values with the x values - var res = []; - for (var i = 0; i < data.length; ++i) { - res.push([i, data[i]]) - } - return res; - } - - var updateInterval = 1000; - var options = { - series: { shadowSize: 1 }, - lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }}, - yaxis: { min: 0, max: 40 }, - colors: ["#a971ff"], - grid: { - tickColor: "#dddddd", - borderWidth: 0 - }, - }; - var plot = $.plot("#realtimechart", [ getData() ], options); - - function update() { - plot.setData([getData()]); - plot.getOptions().yaxes[0].max = curr_max; - plot.setupGrid(); - plot.draw(); - setTimeout(update, updateInterval); - } - update(); -}); - function initfunc( csvay, scroot) { window.csv = csvay; window.scroot = scroot; diff --git a/var/www/templates/Queue_live_Monitoring.html b/var/www/templates/Queue_live_Monitoring.html index 81b322b3..dc278f23 100644 --- a/var/www/templates/Queue_live_Monitoring.html +++ b/var/www/templates/Queue_live_Monitoring.html @@ -1,7 +1,6 @@ - WordsTrend diff --git a/var/www/templates/Trending.html b/var/www/templates/Trending.html index 8928f948..f99a88f9 100644 --- a/var/www/templates/Trending.html +++ b/var/www/templates/Trending.html @@ -14,9 +14,8 @@ - - @@ -56,7 +55,7 @@
- +
- -
+ +
{% include 'trending_graphs/Tldstrending.html' %}
@@ -75,7 +74,7 @@
{% include 'trending_graphs/Wordstrending.html' %} -
+
{% include 'trending_graphs/Protocolstrending.html' %}
@@ -98,27 +97,27 @@ current_displayed_graph = new Graph(pannel, path, header_size); setTimeout(function() { current_displayed_graph.set_Visibility(default_display)}, 300); }, 'text'); - + } // When a pannel is shown, create_and_plot. $('.nav-tabs a').on('shown.bs.tab', function(event){ - create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path')); + create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path')); });
diff --git a/var/www/templates/index.html b/var/www/templates/index.html index 551c56b8..f4559450 100644 --- a/var/www/templates/index.html +++ b/var/www/templates/index.html @@ -14,11 +14,10 @@ - - - -