diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..175d1d68 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +*.git +*.md diff --git a/.travis.yml b/.travis.yml index 1ce79b41..554d0967 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,27 +5,7 @@ python: sudo: required -dist: trusty - -addons: - apt: - packages: - # General dependencies - - python-pip - - python-virtualenv - - python-dev - - g++ - - python-tk - - unzip - - libsnappy-dev - # Needed for bloomfilters - - libssl-dev - - python-numpy - - libfreetype6-dev - # Leveldb - - libgmp-dev - - libev-dev - - cmake +cache: pip env: - AIL_HOME=$TRAVIS_BUILD_DIR AIL_BIN=$TRAVIS_BUILD_DIR/bin/ \ @@ -35,49 +15,7 @@ env: install: - - pip install -U pip - # DNS - - sudo apt-get install -y libadns1 libadns1-dev screen - # required for mathplotlib - - test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/ - - pip install distribute - # Redis - - test ! -d redis/ && git clone https://github.com/antirez/redis.git - - pushd redis - - git checkout 3.2 - - make - - popd - # Redis leveldb - - test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git - - pushd redis-leveldb/ - - git submodule init - - git submodule update - - make - - popd - # Faup - - test ! -d faup && git clone https://github.com/stricaud/faup.git - - pushd faup/ - - test ! -d build && mkdir build - - cd build - - cmake .. && make - - sudo make install - - echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf - - sudo ldconfig - - popd - # PyFaup - - pushd faup/src/lib/bindings/python/ - - python setup.py install - - popd - # Set config - - cp bin/packages/config.cfg.sample bin/packages/config.cfg - - mkdir -p $AIL_HOME/{PASTES,Blooms,dumps} - - mkdir -p $AIL_HOME/LEVEL_DB_DATA/{2016,2015,2014,2013} - - pip install -r pip_packages_requirement.txt - - python -m textblob.download_corpora - - pushd var/www/ - - ./update_thirdparty.sh - - popd - + - ./installing_deps.sh script: - pushd bin diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..f742e9ea --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:14.04 + +RUN mkdir /opt/AIL && apt-get update -y \ + && apt-get install git python-dev build-essential \ + libffi-dev libssl-dev libfuzzy-dev wget -y +ADD . /opt/AIL +WORKDIR /opt/AIL +RUN ./installing_deps.sh && cd var/www/ && ./update_thirdparty.sh +CMD bash docker_start.sh diff --git a/README.md b/README.md index 95398f3b..f5cdb7b4 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ AIL AIL framework - Framework for Analysis of Information Leaks -AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services. AIL framework is flexible and can be extended to support other functionalities to mine sensitive information. +AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services or unstructured data streams. AIL framework is flexible and can be extended to support other functionalities to mine sensitive information. ![Dashboard](./doc/screenshots/dashboard.png?raw=true "AIL framework dashboard") @@ -38,7 +38,7 @@ Terms manager and occurence ![Term-Plot](./doc/screenshots/terms-plot.png?raw=true "AIL framework termPlot") -AIL framework screencast: https://www.youtube.com/watch?v=9idfHCIMzBY +[AIL framework screencast](https://www.youtube.com/watch?v=1_ZrZkRKmNo) Features -------- @@ -54,7 +54,7 @@ Features * A full-text indexer module to index unstructured information * Modules and web statistics * Global sentiment analysis for each providers based on nltk vader module -* Terms tracking and occurence +* Terms tracking and occurrence * Many more modules for extracting phone numbers, credentials and others Installation diff --git a/bin/Attribute.py b/bin/Attributes.py similarity index 100% rename from bin/Attribute.py rename to bin/Attributes.py diff --git a/bin/Browse_warning_paste.py b/bin/BrowseWarningPaste.py similarity index 97% rename from bin/Browse_warning_paste.py rename to bin/BrowseWarningPaste.py index 49444979..01704902 100755 --- a/bin/Browse_warning_paste.py +++ b/bin/BrowseWarningPaste.py @@ -24,7 +24,7 @@ if __name__ == "__main__": publisher.port = 6380 publisher.channel = "Script" - config_section = 'Browse_warning_paste' + config_section = 'BrowseWarningPaste' p = Process(config_section) diff --git a/bin/CreditCard.py b/bin/CreditCards.py similarity index 97% rename from bin/CreditCard.py rename to bin/CreditCards.py index 430c6d7a..04ce9c62 100755 --- a/bin/CreditCard.py +++ b/bin/CreditCards.py @@ -66,7 +66,7 @@ if __name__ == "__main__": publisher.warning('{}Checked {} valid number(s)'.format( to_print, len(creditcard_set))) #Send to duplicate - p.populate_set_out(filepath, 'Duplicate') + p.populate_set_out(filename, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste') else: diff --git a/bin/Curve_manage_top_sets.py b/bin/CurveManageTopSets.py similarity index 84% rename from bin/Curve_manage_top_sets.py rename to bin/CurveManageTopSets.py index 34c1c238..8f316333 100755 --- a/bin/Curve_manage_top_sets.py +++ b/bin/CurveManageTopSets.py @@ -22,8 +22,8 @@ from pubsublogger import publisher from packages import lib_words import datetime import calendar - -from Helper import Process +import os +import ConfigParser # Config Variables Refresh_rate = 60*5 #sec @@ -96,13 +96,19 @@ if __name__ == '__main__': # Script is the default channel used for the modules. publisher.channel = 'Script' - config_section = 'CurveManageTopSets' - p = Process(config_section) + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + + cfg = ConfigParser.ConfigParser() + cfg.read(configfile) server_term = redis.StrictRedis( - host=p.config.get("Redis_Level_DB_TermFreq", "host"), - port=p.config.get("Redis_Level_DB_TermFreq", "port"), - db=p.config.get("Redis_Level_DB_TermFreq", "db")) + host=cfg.get("Redis_Level_DB_TermFreq", "host"), + port=cfg.getint("Redis_Level_DB_TermFreq", "port"), + db=cfg.getint("Redis_Level_DB_TermFreq", "db")) publisher.info("Script Curve_manage_top_set started") @@ -113,11 +119,6 @@ if __name__ == '__main__': while True: # Get one message from the input queue (module only work if linked with a queue) - message = p.get_from_set() - if message is None: - publisher.debug("{} queue is empty, waiting".format(config_section)) - print 'sleeping' - time.sleep(Refresh_rate) # sleep a long time then manage the set - manage_top_set() - continue + time.sleep(Refresh_rate) # sleep a long time then manage the set + manage_top_set() diff --git a/bin/Duplicate_ssdeep.py b/bin/Duplicate_ssdeep.py deleted file mode 100755 index 1b173eca..00000000 --- a/bin/Duplicate_ssdeep.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python2 -# -*-coding:UTF-8 -* - -""" -The Duplicate module -==================== - -This huge module is, in short term, checking duplicates. - -Requirements: -------------- - - -""" -import redis -import os -import time -import datetime -import json -import ssdeep -from packages import Paste -from pubsublogger import publisher - -from Helper import Process - -if __name__ == "__main__": - publisher.port = 6380 - publisher.channel = "Script" - - config_section = 'Duplicates' - save_dico_and_reload = 1 #min - time_1 = time.time() - flag_reload_from_disk = True - flag_write_to_disk = False - - p = Process(config_section) - - # REDIS # - # DB OBJECT & HASHS ( DISK ) - # FIXME increase flexibility - dico_redis = {} - for year in xrange(2013, datetime.date.today().year+1): - for month in xrange(0, 16): - dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis( - host=p.config.get("Redis_Level_DB", "host"), port=year, - db=month) - #print("dup: "+str(year)+str(month).zfill(2)+"\n") - - # FUNCTIONS # - publisher.info("Script duplicate started") - - dicopath = os.path.join(os.environ['AIL_HOME'], - p.config.get("Directories", "dicofilters")) - - dico_path_set = set() - while True: - try: - hash_dico = {} - dupl = [] - - x = time.time() - - message = p.get_from_set() - if message is not None: - path = message - PST = Paste.Paste(path) - else: - publisher.debug("Script Attribute is idling 10s") - time.sleep(10) - continue - - PST._set_p_hash_kind("ssdeep") - - # Assignate the correct redis connexion - r_serv1 = dico_redis[PST.p_date.year + PST.p_date.month] - - # Creating the dicor name: dicoyyyymm - filedicopath = os.path.join(dicopath, 'dico' + PST.p_date.year + - PST.p_date.month) - filedicopath_today = filedicopath - - # Save I/O - if time.time() - time_1 > save_dico_and_reload*60: - flag_write_to_disk = True - - if os.path.exists(filedicopath): - if flag_reload_from_disk == True: - flag_reload_from_disk = False - print 'Reloading' - with open(filedicopath, 'r') as fp: - today_dico = json.load(fp) - else: - today_dico = {} - with open(filedicopath, 'w') as fp: - json.dump(today_dico, fp) - - # For now, just use monthly dico - dico_path_set.add(filedicopath) - - # UNIQUE INDEX HASHS TABLE - yearly_index = str(datetime.date.today().year)+'00' - r_serv0 = dico_redis[yearly_index] - r_serv0.incr("current_index") - index = r_serv0.get("current_index")+str(PST.p_date) - - # For each dico - opened_dico = [] - for dico in dico_path_set: - # Opening dico - if dico == filedicopath_today: - opened_dico.append([dico, today_dico]) - else: - with open(dico, 'r') as fp: - opened_dico.append([dico, json.load(fp)]) - - - #retrieve hash from paste - paste_hash = PST._get_p_hash() - - # Go throught the Database of the dico (of the month) - threshold_dup = 99 - for dico_name, dico in opened_dico: - for dico_key, dico_hash in dico.items(): - percent = ssdeep.compare(dico_hash, paste_hash) - if percent > threshold_dup: - db = dico_name[-6:] - # Go throught the Database of the dico filter (month) - r_serv_dico = dico_redis[db] - - # index of paste - index_current = r_serv_dico.get(dico_hash) - paste_path = r_serv_dico.get(index_current) - if paste_path != None: - hash_dico[dico_hash] = (paste_path, percent) - - #print 'comparing: ' + str(dico_hash[:20]) + ' and ' + str(paste_hash[:20]) + ' percentage: ' + str(percent) - print ' '+ PST.p_path[44:] +', '+ paste_path[44:] + ', ' + str(percent) - - # Add paste in DB to prevent its analyse twice - # HASHTABLES PER MONTH (because of r_serv1 changing db) - r_serv1.set(index, PST.p_path) - r_serv1.sadd("INDEX", index) - # Adding the hash in Redis - r_serv1.set(paste_hash, index) - r_serv1.sadd("HASHS", paste_hash) - ##################### Similarity found ####################### - - # if there is data in this dictionnary - if len(hash_dico) != 0: - for dico_hash, paste_tuple in hash_dico.items(): - paste_path, percent = paste_tuple - dupl.append((paste_path, percent)) - - # Creating the object attribute and save it. - to_print = 'Duplicate;{};{};{};'.format( - PST.p_source, PST.p_date, PST.p_name) - if dupl != []: - PST.__setattr__("p_duplicate", dupl) - PST.save_attribute_redis("p_duplicate", dupl) - publisher.info('{}Detected {}'.format(to_print, len(dupl))) - print '{}Detected {}'.format(to_print, len(dupl)) - - y = time.time() - - publisher.debug('{}Processed in {} sec'.format(to_print, y-x)) - - - # Adding the hash in the dico of the month - today_dico[index] = paste_hash - - if flag_write_to_disk: - time_1 = time.time() - flag_write_to_disk = False - flag_reload_from_disk = True - print 'writing' - with open(filedicopath, 'w') as fp: - json.dump(today_dico, fp) - except IOError: - to_print = 'Duplicate;{};{};{};'.format( - PST.p_source, PST.p_date, PST.p_name) - print "CRC Checksum Failed on :", PST.p_path - publisher.error('{}CRC Checksum Failed'.format(to_print)) diff --git a/bin/Duplicate_ssdeep_v2.py b/bin/Duplicates.py similarity index 100% rename from bin/Duplicate_ssdeep_v2.py rename to bin/Duplicates.py diff --git a/bin/Duplicate.py b/bin/Duplicates_old.py similarity index 100% rename from bin/Duplicate.py rename to bin/Duplicates_old.py diff --git a/bin/Helper.py b/bin/Helper.py index 78a1c94f..05b73bf3 100755 --- a/bin/Helper.py +++ b/bin/Helper.py @@ -16,6 +16,7 @@ import ConfigParser import os import zmq import time +import datetime import json @@ -107,6 +108,7 @@ class Process(object): self.modules = ConfigParser.ConfigParser() self.modules.read(modulesfile) self.subscriber_name = conf_section + self.pubsub = None if self.modules.has_section(conf_section): self.pubsub = PubSub() @@ -117,6 +119,9 @@ class Process(object): port=self.config.get('RedisPubSub', 'port'), db=self.config.get('RedisPubSub', 'db')) + self.moduleNum = os.getpid() + + def populate_set_in(self): # monoproc src = self.modules.get(self.subscriber_name, 'subscribe') @@ -132,7 +137,30 @@ class Process(object): in_set = self.subscriber_name + 'in' self.r_temp.hset('queues', self.subscriber_name, int(self.r_temp.scard(in_set))) - return self.r_temp.spop(in_set) + message = self.r_temp.spop(in_set) + timestamp = int(time.mktime(datetime.datetime.now().timetuple())) + dir_name = os.environ['AIL_HOME']+self.config.get('Directories', 'pastes') + + if message is None: + return None + + else: + try: + if ".gz" in message: + path = message.split(".")[-2].split("/")[-1] + else: + path = "?" + value = str(timestamp) + ", " + path + self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value) + self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum)) + return message + + except: + path = "?" + value = str(timestamp) + ", " + path + self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value) + self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum)) + return message def populate_set_out(self, msg, channel=None): # multiproc diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index fef9c3ce..d7c31472 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -112,33 +112,35 @@ function launching_scripts { echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT + screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x' + sleep 0.1 screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate_ssdeep_v2.py; read x' + screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x' + screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x' + screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x' + screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "CreditCard" bash -c './CreditCard.py; read x' + screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "Url" bash -c './Url.py; read x' + screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "Curve_topsets_manager" bash -c './Curve_manage_top_sets.py; read x' + screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' sleep 0.1 @@ -156,9 +158,9 @@ function launching_scripts { sleep 0.1 screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' + screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "SentimentAnalyser" bash -c './SentimentAnalyser.py; read x' + screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' } diff --git a/bin/Line.py b/bin/Lines.py similarity index 100% rename from bin/Line.py rename to bin/Lines.py diff --git a/bin/ModuleInformation.py b/bin/ModuleInformation.py new file mode 100755 index 00000000..1538f57a --- /dev/null +++ b/bin/ModuleInformation.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +''' + +This module can be use to see information of running modules. +These information are logged in "logs/moduleInfo.log" + +It can also try to manage them by killing inactive one. +However, it does not support mutliple occurence of the same module +(It will kill the first one obtained by get) + + +''' + +import time +import datetime +import redis +import os +import signal +import argparse +from subprocess import PIPE, Popen +import ConfigParser +import json +from terminaltables import AsciiTable +import textwrap + +# CONFIG VARIABLES +threshold_stucked_module = 60*60*1 #1 hour +log_filename = "../logs/moduleInfo.log" +command_search_pid = "ps a -o pid,cmd | grep {}" +command_search_name = "ps a -o pid,cmd | grep {}" +command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\"" + + +def getPid(module): + p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) + for line in p.stdout: + splittedLine = line.split() + if 'python2' in splittedLine: + return int(splittedLine[0]) + else: + return None + +def clearRedisModuleInfo(): + for k in server.keys("MODULE_*"): + server.delete(k) + +def cleanRedis(): + for k in server.keys("MODULE_TYPE_*"): + moduleName = k[12:].split('_')[0] + for pid in server.smembers(k): + flag_pid_valid = False + proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) + for line in proc.stdout: + splittedLine = line.split() + if ('python2' in splittedLine or 'python' in splittedLine) and "./"+moduleName+".py" in splittedLine: + flag_pid_valid = True + + if not flag_pid_valid: + print flag_pid_valid, 'cleaning', pid, 'in', k + server.srem(k, pid) + time.sleep(5) + + +def kill_module(module): + print '' + print '-> trying to kill module:', module + + pid = getPid(module) + if pid is not None: + os.kill(pid, signal.SIGUSR1) + time.sleep(1) + if getPid(module) is None: + print module, 'has been killed' + print 'restarting', module, '...' + p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) + + else: + print 'killing failed, retrying...' + time.sleep(3) + os.kill(pid, signal.SIGUSR1) + time.sleep(1) + if getPid(module) is None: + print module, 'has been killed' + print 'restarting', module, '...' + p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) + else: + print 'killing failed!' + time.sleep(7) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.') + parser.add_argument('-r', '--refresh', type=int, required=False, default=1, help='Refresh rate') + parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)') + parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)') + + args = parser.parse_args() + + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + + cfg = ConfigParser.ConfigParser() + cfg.read(configfile) + + # REDIS # + server = redis.StrictRedis( + host=cfg.get("Redis_Queues", "host"), + port=cfg.getint("Redis_Queues", "port"), + db=cfg.getint("Redis_Queues", "db")) + + if args.clear == 1: + clearRedisModuleInfo() + + lastTime = datetime.datetime.now() + + module_file_array = set() + with open('../doc/all_modules.txt', 'r') as module_file: + for line in module_file: + module_file_array.add(line[:-1]) + + while True: + + all_queue = set() + printarray1 = [] + printarray2 = [] + printarray3 = [] + for queue, card in server.hgetall("queues").iteritems(): + all_queue.add(queue) + key = "MODULE_" + queue + "_" + keySet = "MODULE_TYPE_" + queue + + for moduleNum in server.smembers(keySet): + value = server.get(key + str(moduleNum)) + if value is not None: + timestamp, path = value.split(", ") + if timestamp is not None and path is not None: + startTime_readable = datetime.datetime.fromtimestamp(int(timestamp)) + processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0] + + if int(card) > 0: + if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module: + log = open(log_filename, 'a') + log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n") + if args.autokill == 1: + kill_module(queue) + + printarray1.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)]) + + else: + printarray2.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)]) + + for curr_queue in module_file_array: + if curr_queue not in all_queue: + printarray3.append([curr_queue, "Not running"]) + + printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) + printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) + printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"]) + printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"]) + printarray3.insert(0,["Queue", "State"]) + + os.system('clear') + t1 = AsciiTable(printarray1, title="Working queues") + t1.column_max_width(1) + if not t1.ok: + longest_col = t1.column_widths.index(max(t1.column_widths)) + max_length_col = t1.column_max_width(longest_col) + if max_length_col > 0: + for i, content in enumerate(t1.table_data): + if len(content[longest_col]) > max_length_col: + temp = '' + for l in content[longest_col].splitlines(): + if len(l) > max_length_col: + temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n' + else: + temp += l + '\n' + content[longest_col] = temp.strip() + t1.table_data[i] = content + + t2 = AsciiTable(printarray2, title="Idling queues") + t2.column_max_width(1) + if not t2.ok: + longest_col = t2.column_widths.index(max(t2.column_widths)) + max_length_col = t2.column_max_width(longest_col) + if max_length_col > 0: + for i, content in enumerate(t2.table_data): + if len(content[longest_col]) > max_length_col: + temp = '' + for l in content[longest_col].splitlines(): + if len(l) > max_length_col: + temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n' + else: + temp += l + '\n' + content[longest_col] = temp.strip() + t2.table_data[i] = content + + t3 = AsciiTable(printarray3, title="Not running queues") + t3.column_max_width(1) + + print t1.table + print '\n' + print t2.table + print '\n' + print t3.table + + if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5: + lastTime = datetime.datetime.now() + cleanRedis() + time.sleep(args.refresh) diff --git a/bin/SentimentAnalyser.py b/bin/SentimentAnalysis.py similarity index 100% rename from bin/SentimentAnalyser.py rename to bin/SentimentAnalysis.py diff --git a/bin/Url.py b/bin/Web.py similarity index 100% rename from bin/Url.py rename to bin/Web.py diff --git a/bin/WebStats.py b/bin/WebStats.py index 6fdd9ee3..d8ff0876 100755 --- a/bin/WebStats.py +++ b/bin/WebStats.py @@ -66,23 +66,7 @@ def compute_progression(server, field_name, num_day, url_parsed): # filter if (keyword_total_sum > threshold_total_sum) and (keyword_increase > threshold_increase): - if server.sismember(redis_progression_name_set, keyword): #if keyword is in the set - server.hset(redis_progression_name, keyword, keyword_increase) #update its value - - elif (server.scard(redis_progression_name_set) < max_set_cardinality): - server.sadd(redis_progression_name_set, keyword) - - else: #not in the set - #Check value for all members - member_set = [] - for keyw in server.smembers(redis_progression_name_set): - member_set.append((keyw, int(server.hget(redis_progression_name, keyw)))) - print member_set - member_set.sort(key=lambda tup: tup[1]) - if member_set[0][1] < keyword_increase: - #remove min from set and add the new one - server.srem(redis_progression_name_set, member_set[0]) - server.sadd(redis_progression_name_set, keyword) + server.zadd("z_top_progression_"+field_name, float(keyword_increase), keyword) if __name__ == '__main__': diff --git a/bin/launch_scripts.sh b/bin/launch_scripts.sh index ad55244a..694f0138 100755 --- a/bin/launch_scripts.sh +++ b/bin/launch_scripts.sh @@ -8,50 +8,52 @@ sleep 0.1 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT -screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate_ssdeep_v2.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "CreditCard" bash -c './CreditCard.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Url" bash -c './Url.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Curve_topsets_manager" bash -c './Curve_manage_top_sets.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "SentimentAnalyser" bash -c './SentimentAnalyser.py; read x' + screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py index d91018bd..6d18e846 100755 --- a/bin/packages/Paste.py +++ b/bin/packages/Paste.py @@ -264,7 +264,7 @@ class Paste(object): def _get_p_duplicate(self): self.p_duplicate = self.store.hget(self.p_path, "p_duplicate") - return self.p_duplicate if self.p_duplicate is not None else [] + return self.p_duplicate if self.p_duplicate is not None else '[]' def save_all_attributes_redis(self, key=None): """ diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 7434def8..53bbb2a6 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -34,7 +34,7 @@ subscribe = Redis_Global publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve [CreditCards] -subscribe = Redis_CreditCard +subscribe = Redis_CreditCards publish = Redis_Duplicate,Redis_ModuleStats,Redis_BrowseWarningPaste [Mail] @@ -63,7 +63,7 @@ publish = Redis_BrowseWarningPaste,Redis_Duplicate [ModuleStats] subscribe = Redis_ModuleStats -[Browse_warning_paste] +[BrowseWarningPaste] subscribe = Redis_BrowseWarningPaste #[send_to_queue] @@ -88,9 +88,6 @@ publish = Redis_BrowseWarningPaste,Redis_Duplicate subscribe = Redis_Global publish = Redis_Duplicate,Redis_BrowseWarningPaste -[SourceCode] -subscribe = Redis_SourceCode - [Keys] subscribe = Redis_Global publish = Redis_Duplicate,Redis_BrowseWarningPaste diff --git a/doc/generate_graph_data.py b/doc/generate_graph_data.py new file mode 100755 index 00000000..c1b4b98e --- /dev/null +++ b/doc/generate_graph_data.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +content = "" +modules = {} +all_modules = [] +curr_module = "" +streamingPub = {} +streamingSub = {} + +with open('../bin/packages/modules.cfg', 'r') as f: + for line in f: + if line[0] != '#': + if line[0] == '[': + curr_name = line.replace('[','').replace(']','').replace('\n', '').replace(' ', '') + all_modules.append(curr_name) + modules[curr_name] = {'sub': [], 'pub': []} + curr_module = curr_name + elif curr_module != "": # searching for sub or pub + if line.startswith("subscribe"): + curr_subscribers = [w for w in line.replace('\n', '').replace(' ', '').split('=')[1].split(',')] + modules[curr_module]['sub'] = curr_subscribers + for sub in curr_subscribers: + streamingSub[sub] = curr_module + + elif line.startswith("publish"): + curr_publishers = [w for w in line.replace('\n', '').replace(' ', '').split('=')[1].split(',')] + modules[curr_module]['pub'] = curr_publishers + for pub in curr_publishers: + streamingPub[pub] = curr_module + else: + continue + + output_set_graph = set() + with open('all_modules.txt', 'w') as f2: + for e in all_modules: + f2.write(e+"\n") + + for module in modules.keys(): + for stream_in in modules[module]['sub']: + if stream_in not in streamingPub.keys(): + output_set_graph.add("\"" + stream_in + "\" [color=darkorange1] ;\n") + output_set_graph.add("\"" + stream_in + "\"" + "->" + module + ";\n") + else: + output_set_graph.add("\"" + streamingPub[stream_in] + "\"" + "->" + module + ";\n") + + for stream_out in modules[module]['pub']: + if stream_out not in streamingSub.keys(): + output_set_graph.add("\"" + stream_out + "\" [color=darkorange1] ;\n") + output_set_graph.add("\"" + stream_out + "\"" + "->" + module + ";\n") + else: + output_set_graph.add("\"" + module + "\"" + "->" + streamingSub[stream_out] + ";\n") + + + output_text_graph = "" + output_text_graph += "digraph unix {\n"\ + "graph [pad=\"0.5\"];\n"\ + "size=\"25,25\";\n"\ + "node [color=lightblue2, style=filled];\n" + + for elem in output_set_graph: + output_text_graph += elem + + output_text_graph += "}" + print output_text_graph diff --git a/doc/generate_modules_data_flow_graph.sh b/doc/generate_modules_data_flow_graph.sh new file mode 100755 index 00000000..90a578e0 --- /dev/null +++ b/doc/generate_modules_data_flow_graph.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python generate_graph_data.py | dot -T png -o module-data-flow.png diff --git a/doc/module-data-flow.png b/doc/module-data-flow.png new file mode 100644 index 00000000..73ed3b1d Binary files /dev/null and b/doc/module-data-flow.png differ diff --git a/doc/screenshots/sentiment.png b/doc/screenshots/sentiment.png deleted file mode 100644 index c4edee90..00000000 Binary files a/doc/screenshots/sentiment.png and /dev/null differ diff --git a/doc/screenshots/terms-plot.png b/doc/screenshots/terms-plot.png index 32647906..4d33466d 100644 Binary files a/doc/screenshots/terms-plot.png and b/doc/screenshots/terms-plot.png differ diff --git a/doc/screenshots/terms-top.png b/doc/screenshots/terms-top.png index e833ac35..fc75aed8 100644 Binary files a/doc/screenshots/terms-top.png and b/doc/screenshots/terms-top.png differ diff --git a/doc/screenshots/trending-module.png b/doc/screenshots/trending-module.png index 1330ead8..48304059 100644 Binary files a/doc/screenshots/trending-module.png and b/doc/screenshots/trending-module.png differ diff --git a/doc/screenshots/trending-web.png b/doc/screenshots/trending-web.png index cd7e510b..a98ff47f 100644 Binary files a/doc/screenshots/trending-web.png and b/doc/screenshots/trending-web.png differ diff --git a/docker_start.sh b/docker_start.sh new file mode 100755 index 00000000..f4bf25bb --- /dev/null +++ b/docker_start.sh @@ -0,0 +1,75 @@ +source ./AILENV/bin/activate +cd bin + +export PATH=$AIL_HOME:$PATH +export PATH=$AIL_REDIS:$PATH +export PATH=$AIL_LEVELDB:$PATH +export AILENV=/opt/AIL + +conf_dir="${AIL_HOME}/configs/" + +screen -dmS "Redis" +screen -S "Redis" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x' +screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x' +screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x' + +# For Words and curves +sleep 0.1 +screen -S "Redis" -X screen -t "6382" bash -c 'redis-server '$conf_dir'6382.conf ; read x' + +#Want to launch more level_db? +lvdbhost='127.0.0.1' +lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" +db1_y='2013' +db2_y='2014' +db3_y='2016' +db4_y='3016' +nb_db=13 + +screen -dmS "LevelDB" +#Add lines here with appropriates options. +screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x' +screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x' +screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x' + +# For Curve +screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x' + + +screen -dmS "Logging" +screen -S "Logging" -X screen -t "LogQueue" bash -c 'log_subscriber -p 6380 -c Queuing -l ../logs/; read x' +screen -S "Logging" -X screen -t "LogScript" bash -c 'log_subscriber -p 6380 -c Script -l ../logs/; read x' + +screen -dmS "Queue" +screen -S "Queue" -X screen -t "Queues" bash -c './launch_queues.py; read x' + +screen -dmS "Script" +screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x' +screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' +screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' +screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x' +screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x' +screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x' +screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' +screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x' +screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x' +screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x' +screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x' +screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x' +screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x' +screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' +screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x' +screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' +screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' +screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x' +screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x' +screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' +screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' +screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x' +screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' +screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x' +screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' + +cd $AILENV +cd var/www/ +python Flask_server.py diff --git a/installing_deps.sh b/installing_deps.sh index 9117dd75..b3df3221 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -6,20 +6,24 @@ set -x sudo apt-get update sudo apt-get install python-pip python-virtualenv python-dev libfreetype6-dev \ - screen g++ python-tk unzip libsnappy-dev cmake + screen g++ python-tk unzip libsnappy-dev cmake -y #Needed for bloom filters -sudo apt-get install libssl-dev libfreetype6-dev python-numpy +sudo apt-get install libssl-dev libfreetype6-dev python-numpy -y # DNS deps -sudo apt-get install libadns1 libadns1-dev +sudo apt-get install libadns1 libadns1-dev -y #Needed for redis-lvlDB -sudo apt-get install libev-dev libgmp-dev +sudo apt-get install libev-dev libgmp-dev -y + +#Need for generate-data-flow graph +sudo apt-get install graphviz -y #needed for mathplotlib -test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/ sudo easy_install -U distribute +# ssdeep +sudo apt-get install libfuzzy-dev # REDIS # test ! -d redis/ && git clone https://github.com/antirez/redis.git @@ -29,7 +33,7 @@ make popd # Faup -test ! -d faup && git clone https://github.com/stricaud/faup.git +test ! -d faup/ && git clone https://github.com/stricaud/faup.git pushd faup/ test ! -d build && mkdir build cd build @@ -43,6 +47,10 @@ popd test ! -d tlsh && git clone git://github.com/trendmicro/tlsh.git pushd tlsh/ ./make.sh +pushd build/release/ +sudo make install +sudo ldconfig +popd popd # REDIS LEVEL DB # @@ -57,22 +65,30 @@ if [ ! -f bin/packages/config.cfg ]; then cp bin/packages/config.cfg.sample bin/packages/config.cfg fi -virtualenv AILENV +pushd var/www/ +./update_thirdparty.sh +popd -echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate -echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate -echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate -echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate -echo export AIL_LEVELDB=$(pwd)/redis-leveldb/ >> ./AILENV/bin/activate +if [ -z "$VIRTUAL_ENV" ]; then -. ./AILENV/bin/activate + virtualenv AILENV + + echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate + echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate + echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate + echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate + echo export AIL_LEVELDB=$(pwd)/redis-leveldb/ >> ./AILENV/bin/activate + + . ./AILENV/bin/activate + +fi mkdir -p $AIL_HOME/{PASTES,Blooms,dumps} mkdir -p $AIL_HOME/LEVEL_DB_DATA/2016 mkdir -p $AIL_HOME/LEVEL_DB_DATA/3016 pip install -U pip -pip install -r pip_packages_requirement.txt +pip install -U -r pip_packages_requirement.txt # Pyfaup pushd faup/src/lib/bindings/python/ @@ -81,6 +97,7 @@ popd # Py tlsh pushd tlsh/py_ext +git checkout a67c69b0cdfd168c62c159d41b8a3612ee2b0df1 # temporary, latest commit breaks the python module python setup.py build python setup.py install diff --git a/pip_packages_requirement.txt b/pip_packages_requirement.txt index bd734175..f6602653 100644 --- a/pip_packages_requirement.txt +++ b/pip_packages_requirement.txt @@ -10,6 +10,7 @@ textblob numpy matplotlib networkx +terminaltables #Tokeniser nltk @@ -39,7 +40,7 @@ pycountry PySocks #ASN lookup requirements -http://adns-python.googlecode.com/files/adns-python-1.2.1.tar.gz +https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz https://github.com/trolldbois/python-cymru-services/archive/master.zip https://github.com/saffsd/langid.py/archive/master.zip diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 45efd8f1..f3f9d71a 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -81,48 +81,31 @@ def event_stream(): def get_queues(r): # We may want to put the llen in a pipeline to do only one query. - return [(queue, int(card)) for queue, card in - r.hgetall("queues").iteritems()] + newData = [] + for queue, card in r.hgetall("queues").iteritems(): + key = "MODULE_" + queue + "_" + keySet = "MODULE_TYPE_" + queue + + for moduleNum in r.smembers(keySet): + + value = r.get(key + str(moduleNum)) + if value is not None: + timestamp, path = value.split(", ") + if timestamp is not None: + startTime_readable = datetime.datetime.fromtimestamp(int(timestamp)) + processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0] + seconds = int((datetime.datetime.now() - startTime_readable).total_seconds()) + newData.append( (queue, card, seconds, moduleNum) ) + else: + newData.append( (queue, cards, 0, moduleNum) ) + + return newData def list_len(s): return len(s) app.jinja_env.filters['list_len'] = list_len -def parseStringToList(the_string): - strList = "" - elemList = [] - for c in the_string: - if c != ']': - if c != '[' and c !=' ' and c != '"': - strList += c - else: - the_list = strList.split(',') - if len(the_list) == 3: - elemList = elemList + the_list - elif len(the_list) == 2: - elemList.append(the_list) - elif len(the_list) > 1: - elemList.append(the_list[1:]) - strList = "" - return elemList - -def parseStringToList2(the_string): - if the_string == []: - return [] - else: - res = [] - tab_str = the_string.split('], [') - tab_str[0] = tab_str[0][1:]+']' - tab_str[len(tab_str)-1] = '['+tab_str[len(tab_str)-1][:-1] - res.append(parseStringToList(tab_str[0])) - for i in range(1, len(tab_str)-2): - tab_str[i] = '['+tab_str[i]+']' - res.append(parseStringToList(tab_str[i])) - if len(tab_str) > 1: - res.append(parseStringToList(tab_str[len(tab_str)-1])) - return res - def showpaste(content_range): requested_path = request.args.get('paste', '') @@ -136,7 +119,7 @@ def showpaste(content_range): p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content().decode('utf-8', 'ignore') - p_duplicate_full_list = parseStringToList2(paste._get_p_duplicate()) + p_duplicate_full_list = json.loads(paste._get_p_duplicate()) p_duplicate_list = [] p_simil_list = [] p_hashtype_list = [] @@ -160,7 +143,7 @@ def showpaste(content_range): hash_types = [] comp_vals = [] for i in indices: - hash_types.append(p_duplicate_full_list[i][0]) + hash_types.append(p_duplicate_full_list[i][0].encode('utf8')) comp_vals.append(p_duplicate_full_list[i][2]) dup_list_removed.append(i) @@ -267,19 +250,9 @@ def progressionCharts(): return jsonify(bar_values) else: - redis_progression_name = 'top_progression_'+trending_name - redis_progression_name_set = 'top_progression_'+trending_name+'_set' - - # Iterate over element in top_x_set and retreive their value - member_set = [] - for keyw in r_serv_charts.smembers(redis_progression_name_set): - keyw_value = r_serv_charts.hget(redis_progression_name, keyw) - keyw_value = keyw_value if keyw_value is not None else 0 - member_set.append((keyw, int(keyw_value))) - member_set.sort(key=lambda tup: tup[1], reverse=True) - if len(member_set) == 0: - member_set.append(("No relevant data", int(100))) - return jsonify(member_set) + redis_progression_name = "z_top_progression_" + trending_name + keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10) + return jsonify(keyw_value) @app.route("/_moduleCharts", methods=['GET']) def modulesCharts(): @@ -458,7 +431,7 @@ def sentiment_analysis_trending(): return render_template("sentiment_analysis_trending.html") -@app.route("/sentiment_analysis_getplotdata/") +@app.route("/sentiment_analysis_getplotdata/", methods=['GET']) def sentiment_analysis_getplotdata(): # Get the top providers based on number of pastes oneHour = 60*60 @@ -467,19 +440,27 @@ def sentiment_analysis_getplotdata(): dateStart = dateStart.replace(minute=0, second=0, microsecond=0) dateStart_timestamp = calendar.timegm(dateStart.timetuple()) - to_return = {} - range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8) - # if empty, get yesterday top providers - print 'providers_set_'+ get_date_range(1)[1] - range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers - # if still empty, takes from all providers - if range_providers == []: - print 'today provider empty' - range_providers = r_serv_charts.smembers('all_provider_set') + getAllProviders = request.args.get('getProviders') + provider = request.args.get('provider') + allProvider = request.args.get('all') + if getAllProviders == 'True': + if allProvider == "True": + range_providers = r_serv_charts.smembers('all_provider_set') + return jsonify(list(range_providers)) + else: + range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8) + # if empty, get yesterday top providers + range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers + # if still empty, takes from all providers + if range_providers == []: + print 'today provider empty' + range_providers = r_serv_charts.smembers('all_provider_set') + return jsonify(range_providers) - for cur_provider in range_providers: - print cur_provider - cur_provider_name = cur_provider + '_' + elif provider is not None: + to_return = {} + + cur_provider_name = provider + '_' list_date = {} for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour): cur_set_name = cur_provider_name + str(cur_timestamp) @@ -489,9 +470,10 @@ def sentiment_analysis_getplotdata(): cur_value = r_serv_sentiment.get(cur_id) list_value.append(cur_value) list_date[cur_timestamp] = list_value - to_return[cur_provider] = list_date + to_return[provider] = list_date - return jsonify(to_return) + return jsonify(to_return) + return "Bad request" @@ -695,7 +677,6 @@ def terms_plot_tool_data(): else: value_range = [] for timestamp in range(range_start, range_end+oneDay, oneDay): - print timestamp, term value = r_serv_term.hget(timestamp, term) curr_value_range = int(value) if value is not None else 0 value_range.append([timestamp, curr_value_range]) diff --git a/var/www/static/js/indexjavascript.js b/var/www/static/js/indexjavascript.js index f0cd101f..e527aafa 100644 --- a/var/www/static/js/indexjavascript.js +++ b/var/www/static/js/indexjavascript.js @@ -207,7 +207,7 @@ function create_queue_table() { table.appendChild(tableHead); table.appendChild(tableBody); var heading = new Array(); - heading[0] = "Queue Name" + heading[0] = "Queue Name.PID" heading[1] = "Amount" var tr = document.createElement('TR'); tableHead.appendChild(tr); @@ -221,22 +221,31 @@ function create_queue_table() { for(i = 0; i < (glob_tabvar.row1).length;i++){ var tr = document.createElement('TR') - for(j = 0; j < (glob_tabvar.row1[i]).length; j++){ + for(j = 0; j < 2; j++){ var td = document.createElement('TD') - td.appendChild(document.createTextNode(glob_tabvar.row1[i][j])); + var moduleNum = j == 0 ? "." + glob_tabvar.row1[i][3] : ""; + td.appendChild(document.createTextNode(glob_tabvar.row1[i][j] + moduleNum)); tr.appendChild(td) } + // Used to decide the color of the row + // We have glob_tabvar.row1[][j] with: + // - j=0: ModuleName + // - j=1: queueLength + // - j=2: LastProcessedPasteTime + // - j=3: Number of the module belonging in the same category + if (parseInt(glob_tabvar.row1[i][2]) > 60*2 && parseInt(glob_tabvar.row1[i][1]) > 2) + tr.className += " danger"; + else if (parseInt(glob_tabvar.row1[i][2]) > 60*1) + tr.className += " warning"; + else + tr.className += " success"; tableBody.appendChild(tr); } Tablediv.appendChild(table); } -$(document).ready(function () { - if (typeof glob_tabvar == "undefined") - location.reload(); - if (typeof glob_tabvar.row1 == "undefined") - location.reload(); +function load_queues() { var data = []; var data2 = []; var tmp_tab = []; @@ -246,13 +255,17 @@ $(document).ready(function () { var x = new Date(); for (i = 0; i < glob_tabvar.row1.length; i++){ - if (glob_tabvar.row1[i][0] == 'Categ' || glob_tabvar.row1[i][0] == 'Curve'){ - tmp_tab2.push(0); - curves_labels2.push(glob_tabvar.row1[i][0]); + if (glob_tabvar.row1[i][0].split(".")[0] == 'Categ' || glob_tabvar.row1[i][0].split(".")[0] == 'Curve'){ + if (curves_labels2.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) { + tmp_tab2.push(0); + curves_labels2.push(glob_tabvar.row1[i][0].split(".")[0]); + } } else { - tmp_tab.push(0); - curves_labels.push(glob_tabvar.row1[i][0]); + if (curves_labels.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) { + tmp_tab.push(0); + curves_labels.push(glob_tabvar.row1[i][0].split(".")[0]); + } } } tmp_tab.unshift(x); @@ -311,19 +324,29 @@ $(document).ready(function () { update_values(); if($('#button-toggle-queues').prop('checked')){ + $("#queue-color-legend").show(); create_queue_table(); } else{ $("#queueing").html(''); + $("#queue-color-legend").hide(); } + queues_pushed = [] for (i = 0; i < (glob_tabvar.row1).length; i++){ - if (glob_tabvar.row1[i][0] == 'Categ' || glob_tabvar.row1[i][0] == 'Curve'){ - tmp_values2.push(glob_tabvar.row1[i][1]); + if (glob_tabvar.row1[i][0].split(".")[0] == 'Categ' || glob_tabvar.row1[i][0].split(".")[0] == 'Curve'){ + if (queues_pushed.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) { + queues_pushed.push(glob_tabvar.row1[i][0].split(".")[0]); + tmp_values2.push(parseInt(glob_tabvar.row1[i][1])); + } } else { - tmp_values.push(glob_tabvar.row1[i][1]); + if (queues_pushed.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) { + queues_pushed.push(glob_tabvar.row1[i][0].split(".")[0]); + tmp_values.push(parseInt(glob_tabvar.row1[i][1])); + } + } } tmp_values.unshift(x); @@ -362,7 +385,19 @@ $(document).ready(function () { // something went wrong, hide the canvas container document.getElementById('myCanvasContainer').style.display = 'none'; } +} +function manage_undefined() { + if (typeof glob_tabvar == "undefined") + setTimeout(function() { if (typeof glob_tabvar == "undefined") { manage_undefined(); } else { load_queues(); } }, 1000); + else if (typeof glob_tabvar.row1 == "undefined") + setTimeout(function() { if (typeof glob_tabvar.row1 == "undefined") { manage_undefined(); } else { load_queues(); } }, 1000); + else + load_queues(); +} + +$(document).ready(function () { + manage_undefined(); }); diff --git a/var/www/static/js/sentiment_trending.js b/var/www/static/js/sentiment_trending.js index a56723b8..d96c6fbb 100644 --- a/var/www/static/js/sentiment_trending.js +++ b/var/www/static/js/sentiment_trending.js @@ -1,4 +1,42 @@ +/* Functions and config */ +function add_new_graph_today(id) { + return "
" + + "
" + + "Graph "+id+"" + + "Avg" + + "
" + + "
" + + "" + + "" + + "" + + " " + + " " + + "" + + "" + + "
" + + "
" + + "
"; +}; +function add_new_graph_week(id) { + return "
" + + "
" + + "Graph "+id+"" + + "Avg" + + "
" + + "
" + + "" + + "" + + "" + + " " + + " " + + "" + + "" + + "
" + + "
" + + "
"; +} + function generate_offset_to_time(num){ var to_ret = {}; for(i=0; i<=num; i++) { @@ -15,7 +53,7 @@ var to_ret = {}; for(i=day; i>=0; i--){ for(j=0; j<24; j++){ - var t1 =now.getDate()-i + ":"; + var t1 =now.getDate()-i + ":"; var t2 =now.getHours()-(23-j); t2 = t2 < 0 ? 24+t2 : t2; t2 += "h"; @@ -40,34 +78,79 @@ barColor: '#00bf5f', negBarColor: '#f22929', zeroColor: '#ffff00', - + tooltipFormat: '● {{offset:names}}, {{value}} ', }; -$.getJSON("/sentiment_analysis_getplotdata/", - function(data) { - var all_data = []; - var plot_data = []; - var graph_avg = []; - var array_provider = Object.keys(data); - var dates_providers = Object.keys(data[array_provider[0]]); - var dateStart = parseInt(dates_providers[0]); - var oneHour = 60*60; - var oneWeek = oneHour*24*7; +/* Plot and queries */ - var all_graph_day_sum = 0.0; - var all_graph_hour_sum = 0.0; - var all_graph_hour_maxVal = 0.0; - var all_day_avg = 0.0; - var all_day_avg_maxVal = 0.0; +var all_graph_day_sum = 0.0; +var all_graph_hour_sum = 0.0; +var all_graph_hour_sum_minus = 0.0; +var all_graph_hour_maxVal = 0.0; +var all_day_avg = 0.0; +var all_day_avg_maxVal = 0.0; +var graph_avg = []; +var all_data = []; +var provider_already_loaded = []; +var totNumGraph = 0; + +// Query all providers name then launch the query and plot process for each of them. +// When everything is terminated, plot the widgets (Gauge, canvasJS, table) +// input: all - set to 'True' if you take all providers +function draw_page(all) { + $.getJSON("/sentiment_analysis_getplotdata/?getProviders=True&all="+all, + function(data) { + var promises = []; + + var the_length = provider_already_loaded.length == 0 ? 0 : provider_already_loaded.length; + for(i=0; i max_value ? Math.abs(pos-neg) : max_value; if(curr_date >= dateStart+oneWeek-23*oneHour){ - max_value_day = Math.abs(pos-neg) > max_value_day ? Math.abs(pos-neg) : max_value_day; - day_sum += (pos-neg); + max_value_day = Math.abs(pos-neg) > max_value_day ? Math.abs(pos-neg) : max_value_day; day_sum += (pos-neg); day_sum_elem++; } if(curr_date > dateStart+oneWeek-2*oneHour && curr_date <=dateStart+oneWeek-oneHour){ @@ -121,17 +202,16 @@ $.getJSON("/sentiment_analysis_getplotdata/", } all_graph_day_sum += day_sum; all_graph_hour_sum += hour_sum; + all_graph_hour_sum_minus += hour_sum > 0 ? 0 : 1; all_graph_hour_maxVal = Math.abs(hour_sum) > all_graph_hour_maxVal ? Math.abs(hour_sum) : all_graph_hour_maxVal; - var curr_avg = curr_sum / (curr_sum_elem); + var curr_avg = curr_sum / (curr_sum_elem); if(isNaN(curr_avg)) curr_avg = 0.0 - //var curr_avg = curr_sum / (oneWeek/oneHour); - //var curr_avg = curr_sum / (spark_data.length); graph_avg.push([curr_provider, curr_avg]); plot_data.push(spark_data); all_data.push(graph_data); - + sparklineOptions.chartRangeMax = max_value; sparklineOptions.chartRangeMin = -max_value; @@ -141,11 +221,11 @@ $.getJSON("/sentiment_analysis_getplotdata/", var num = graphNum + 1; var placeholder = '.sparkLineStatsWeek' + num; sparklineOptions.barWidth = 2; - $(placeholder).sparkline(plot_data[graphNum], sparklineOptions); + $(placeholder).sparkline(plot_data[0], sparklineOptions); $(placeholder+'t').text(curr_provider); - var curr_avg_text = isNaN(curr_avg) ? "No data" : curr_avg.toFixed(5); + var curr_avg_text = isNaN(curr_avg) ? "No data" : curr_avg.toFixed(5); $(placeholder+'s').text(curr_avg_text); - + sparklineOptions.barWidth = 18; sparklineOptions.tooltipFormat = '● Avg: {{value}} ' $(placeholder+'b').sparkline([curr_avg], sparklineOptions); @@ -169,8 +249,8 @@ $.getJSON("/sentiment_analysis_getplotdata/", // print today - var data_length = plot_data[graphNum].length; - var data_today = plot_data[graphNum].slice(data_length-24, data_length); + var data_length = plot_data[0].length; + var data_today = plot_data[0].slice(data_length-24, data_length); placeholder = '.sparkLineStatsToday' + num; sparklineOptions.barWidth = 14; @@ -199,155 +279,124 @@ $.getJSON("/sentiment_analysis_getplotdata/", $(avgName).addClass("panel-warning") } - }//for loop - - - /* ---------------- Gauge ---------------- */ - var gaugeOptions = { - animateEasing: true, - - elementWidth: 200, - elementHeight: 125, - - arcFillStart: 10, - arcFillEnd: 12, - arcFillTotal: 20, - incTot: 1.0, - - arcBgColorLight: 200, - arcBgColorSat: 0, - arcStrokeFg: 20, - arcStrokeBg: 30, - - colorArcFg: '#FF3300', - animateSpeed: 1, - - }; - // Clone object - var gaugeOptions2 = jQuery.extend(true, {}, gaugeOptions); - var gaugeOptions3 = jQuery.extend(true, {}, gaugeOptions); - - - - gaugeOptions.appendTo = '#gauge_today_last_hour'; - gaugeOptions.dialLabel = 'Last hour'; - gaugeOptions.elementId = 'gauge1'; - var piePercent = (all_graph_hour_sum / 8) / all_graph_hour_maxVal; - gaugeOptions.inc = piePercent; - var gauge_today_last_hour = new FlexGauge(gaugeOptions); - - gaugeOptions2.appendTo = '#gauge_today_last_days'; - gaugeOptions2.dialLabel = 'Today'; - gaugeOptions2.elementId = 'gauge2'; - //piePercent = (all_graph_day_sum / (8*24)) / max_value; - piePercent = (all_day_avg / 8) / all_day_avg_maxVal; - gaugeOptions2.inc = piePercent; - var gauge_today_last_days = new FlexGauge(gaugeOptions2); - - gaugeOptions3.appendTo = '#gauge_week'; - gaugeOptions3.dialLabel = 'Week'; - gaugeOptions3.elementId = 'gauge3'; - - var graph_avg_sum = 0.0; - var temp_max_val = 0.0; - for (i=0; i temp_max_val ? Math.abs(graph_avg[i][1]) : temp_max_val; } - - piePercent = (graph_avg_sum / graph_avg.length) / temp_max_val; - gaugeOptions3.inc = piePercent; - var gauge_today_last_days = new FlexGauge(gaugeOptions3); + ); + return query_plot +} - /* --------- Sort providers -------- */ - graph_avg.sort(function(a, b){return b[1]-a[1]}); +function draw_widgets() { - for (i=1; i<6; i++){ - $('.worst'+i).text(graph_avg[7-(i-1)][0]); - $('.best'+i).text(graph_avg[i-1][0]); - } + /* ---------------- Gauge ---------------- */ + var gaugeOptions = { + animateEasing: true, - /* ----------- CanvasJS ------------ */ + elementWidth: 200, + elementHeight: 125, - var comp_sum_day_pos = 0.0; - var comp_sum_day_neg = 0.0; - var comp_sum_hour_pos = 0.0; - var comp_sum_hour_neg = 0.0; - for(graphNum=0; graphNum<8; graphNum++){ - curr_graphData = all_data[graphNum]; - var gauge_data = curr_graphData.slice(curr_graphData.length-24, curr_graphData.length); - for (i=1; i< gauge_data.length; i++){ - comp_sum_day_pos += gauge_data[i].compoundPos; - comp_sum_day_neg += gauge_data[i].compoundNeg; + arcFillStart: 10, + arcFillEnd: 12, + arcFillTotal: 20, + incTot: 1.0, - if(i == 23){ - comp_sum_hour_pos += gauge_data[i].compoundPos; - comp_sum_hour_neg += gauge_data[i].compoundNeg; - } + arcBgColorLight: 200, + arcBgColorSat: 0, + arcStrokeFg: 20, + arcStrokeBg: 30, + + colorArcFg: '#FF3300', + animateSpeed: 1, + + }; + // Clone object + var gaugeOptions2 = jQuery.extend(true, {}, gaugeOptions); + var gaugeOptions3 = jQuery.extend(true, {}, gaugeOptions); + + + + gaugeOptions.appendTo = '#gauge_today_last_hour'; + gaugeOptions.dialLabel = 'Last hour'; + gaugeOptions.elementId = 'gauge1'; + var piePercent = (all_graph_hour_sum / (totNumGraph - all_graph_hour_sum_minus)) / all_graph_hour_maxVal; + gaugeOptions.inc = piePercent; + var gauge_today_last_hour = new FlexGauge(gaugeOptions); + + gaugeOptions2.appendTo = '#gauge_today_last_days'; + gaugeOptions2.dialLabel = 'Today'; + gaugeOptions2.elementId = 'gauge2'; + piePercent = (all_day_avg / totNumGraph) / all_day_avg_maxVal; + gaugeOptions2.inc = piePercent; + var gauge_today_last_days = new FlexGauge(gaugeOptions2); + + gaugeOptions3.appendTo = '#gauge_week'; + gaugeOptions3.dialLabel = 'Week'; + gaugeOptions3.elementId = 'gauge3'; + + var graph_avg_sum = 0.0; + var temp_max_val = 0.0; + for (i=0; i temp_max_val ? Math.abs(graph_avg[i][1]) : temp_max_val; + } + + piePercent = (graph_avg_sum / graph_avg.length) / temp_max_val; + gaugeOptions3.inc = piePercent; + var gauge_today_last_days = new FlexGauge(gaugeOptions3); + + + /* --------- Sort providers -------- */ + + graph_avg.sort(function(a, b){return b[1]-a[1]}); + + for (i=1; i<6; i++){ + $('.worst'+i).text(graph_avg[7-(i-1)][0]); + $('.best'+i).text(graph_avg[i-1][0]); + } + + /* ----------- CanvasJS ------------ */ + + var comp_sum_day_pos = 0.0; + var comp_sum_day_neg = 0.0; + var comp_sum_hour_pos = 0.0; + var comp_sum_hour_neg = 0.0; + for(graphNum=0; graphNumPositive: {y}", - type: "bar", - color: "green", - dataPoints: [ - {y: comp_sum_hour_pos/8} - ] - }, - { - toolTipContent: "Negative: {y}", - type: "bar", - color: "red", - dataPoints: [ - {y: comp_sum_hour_neg/8} - ] - } - ] - }; - - var chart_canvas1 = new CanvasJS.Chart("bar_today_last_hour", options_canvasJS_1); + } - var options_canvasJS_2 = { - - animationEnabled: true, - axisY: { - tickThickness: 0, - lineThickness: 0, - valueFormatString: " ", - gridThickness: 0 - }, - axisX: { - tickThickness: 0, - lineThickness: 0, - labelFontSize: 0.1, - }, - data: [ + var options_canvasJS_1 = { + + animationEnabled: true, + axisY: { + tickThickness: 0, + lineThickness: 0, + valueFormatString: " ", + gridThickness: 0 + }, + axisX: { + tickThickness: 0, + lineThickness: 0, + labelFontSize: 0.1, + }, + data: [ { toolTipContent: "Positive: {y}", type: "bar", color: "green", dataPoints: [ - {y: comp_sum_day_pos/8} + {y: comp_sum_hour_pos/totNumGraph} ] }, { @@ -355,32 +404,51 @@ $.getJSON("/sentiment_analysis_getplotdata/", type: "bar", color: "red", dataPoints: [ - {y: comp_sum_day_neg/8} + {y: comp_sum_hour_neg/totNumGraph} ] } + ] + }; + + var chart_canvas1 = new CanvasJS.Chart("bar_today_last_hour", options_canvasJS_1); + + var options_canvasJS_2 = { + + animationEnabled: true, + axisY: { + tickThickness: 0, + lineThickness: 0, + valueFormatString: " ", + gridThickness: 0 + }, + axisX: { + tickThickness: 0, + lineThickness: 0, + labelFontSize: 0.1, + }, + data: [ + { + toolTipContent: "Positive: {y}", + type: "bar", + color: "green", + dataPoints: [ + {y: comp_sum_day_pos/totNumGraph} ] - }; - - var chart_canvas2 = new CanvasJS.Chart("bar_today_last_days", options_canvasJS_2); - - chart_canvas1.render(); - chart_canvas2.render(); - - - - } -); - - - - - - - - - - - + }, + { + toolTipContent: "Negative: {y}", + type: "bar", + color: "red", + dataPoints: [ + {y: comp_sum_day_neg/totNumGraph} + ] + } + ] + }; + var chart_canvas2 = new CanvasJS.Chart("bar_today_last_days", options_canvasJS_2); + chart_canvas1.render(); + chart_canvas2.render(); +} diff --git a/var/www/templates/index.html b/var/www/templates/index.html index dc360008..5d8639cf 100644 --- a/var/www/templates/index.html +++ b/var/www/templates/index.html @@ -66,7 +66,18 @@ Display queues
-
+
+ + + + + + + + +
Working queues
 Idling queues
 Stuck queues
+
+
@@ -154,7 +165,6 @@ - diff --git a/var/www/templates/sentiment_analysis_trending.html b/var/www/templates/sentiment_analysis_trending.html index 3a927251..e1788f35 100644 --- a/var/www/templates/sentiment_analysis_trending.html +++ b/var/www/templates/sentiment_analysis_trending.html @@ -113,6 +113,7 @@
+
@@ -128,139 +129,13 @@
-
-
-
- Graph 1 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 2 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 3 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 4 - Avg -
-
- - - - - - - -
-
-
+
+
-
-
-
- Graph 5 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 6 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 7 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 8 - Avg -
-
- - - - - - - -
-
-
+
+
@@ -304,139 +179,13 @@
-
-
-
- Graph 1 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 2 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 3 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 4 - Avg -
-
- - - - - - - -
-
-
+
+
-
-
-
- Graph 5 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 6 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 7 - Avg -
-
- - - - - - - -
-
-
-
-
- Graph 8 - Avg -
-
- - - - - - - -
-
-
+
+
@@ -490,8 +239,6 @@
- -
@@ -502,13 +249,15 @@ @@ -233,15 +232,15 @@ $.getJSON(url, function (data) { if (data.length != 0) { var html_to_add = ""; - html_to_add += ""; + html_to_add += "
"; html_to_add += ""; html_to_add += ""; html_to_add += ""; html_to_add += ""; html_to_add += ""; html_to_add += ""; - html_to_add += ""; - html_to_add += ""; + html_to_add += ""; + html_to_add += ""; html_to_add += ""; html_to_add += ""; html_to_add += ""; @@ -253,8 +252,8 @@ html_to_add += ""; html_to_add += ""; html_to_add += ""; - html_to_add += ""; - html_to_add += ""; + html_to_add += ""; + html_to_add += ""; html_to_add += ""; html_to_add += ""; @@ -264,6 +263,7 @@ $("#mymodalbody").html(html_to_add); $("[data-toggle=popover]").popover(); $("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') ); + $('#modal-table').DataTable(); } else { $("#mymodalbody").html("No paste containing this term has been received yet."); $("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') ); diff --git a/var/www/templates/terms_plot_top.html b/var/www/templates/terms_plot_top.html index fd7a2be8..cf11bc88 100644 --- a/var/www/templates/terms_plot_top.html +++ b/var/www/templates/terms_plot_top.html @@ -12,7 +12,6 @@ - diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index 6ad2744f..c0208bed 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -4,24 +4,28 @@ set -e wget http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js -SBADMIN_VERSION='1.0.4' +SBADMIN_VERSION='3.3.7' rm -rf temp mkdir temp wget https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip +wget https://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}-2.zip unzip temp/${SBADMIN_VERSION}.zip -d temp/ -mv temp/startbootstrap-sb-admin-${SBADMIN_VERSION} temp/sb-admin-2 +unzip temp/${SBADMIN_VERSION}-2.zip -d temp/ +mv temp/startbootstrap-sb-admin-${SBADMIN_VERSION} temp/sb-admin +mv temp/startbootstrap-sb-admin-2-${SBADMIN_VERSION} temp/sb-admin-2 rm -rf ./static/js/plugins -mv temp/sb-admin-2/js/* ./static/js/ +mv temp/sb-admin/js/* ./static/js/ rm -rf ./static/fonts/ ./static/font-awesome/ -mv temp/sb-admin-2/fonts/ ./static/ -mv temp/sb-admin-2/font-awesome/ ./static/ +mv temp/sb-admin/fonts/ ./static/ +mv temp/sb-admin/font-awesome/ ./static/ rm -rf ./static/css/plugins/ -mv temp/sb-admin-2/css/* ./static/css/ +mv temp/sb-admin/css/* ./static/css/ +mv temp/sb-admin-2/dist/css/* ./static/css/ rm -rf temp @@ -39,12 +43,17 @@ wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.pie.js -O ./ wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.time.js -O ./static/js/jquery.flot.time.js wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js -#Ressources for sparkline and canvasJS +#Ressources for sparkline and canvasJS and slider wget http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js +mkdir temp wget http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip unzip temp/canvasjs.zip -d temp/ -mkdir temp mv temp/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js + +wget https://jqueryui.com/resources/download/jquery-ui-1.12.0.zip -O temp/jquery-ui.zip +unzip temp/jquery-ui.zip -d temp/ +mv temp/jquery-ui-1.12.0/jquery-ui.min.js ./static/js/jquery-ui.min.js +mv temp/jquery-ui-1.12.0/jquery-ui.min.css ./static/css/jquery-ui.min.css rm -rf temp mkdir -p ./static/image
SourceDateEncodingSize (Kb)Mime(# lines, Max line length)# linesMax lengthPreview
"+curr_data.date+""+curr_data.encoding+""+curr_data.size+""+curr_data.mime+"("+curr_data.lineinfo[0]+", "+curr_data.lineinfo[1]+")"+curr_data.lineinfo[0]+""+curr_data.lineinfo[1]+"