diff --git a/.gitignore b/.gitignore index 46d87d71..a99c47fd 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ bin/packages/config.cfg # installed files nltk_data/ +doc/all_modules.txt diff --git a/bin/Indexer.py b/bin/Indexer.py index e57b5f4d..29990bfd 100755 --- a/bin/Indexer.py +++ b/bin/Indexer.py @@ -15,13 +15,14 @@ from pubsublogger import publisher from whoosh.index import create_in, exists_in, open_dir from whoosh.fields import Schema, TEXT, ID +import shutil import os from os.path import join, getsize from Helper import Process # Config variable -TIME_WAIT = 1.0 #sec +TIME_WAIT = 60*15 #sec # return in bytes def check_index_size(baseindexpath, indexname): @@ -32,11 +33,9 @@ def check_index_size(baseindexpath, indexname): return cur_sum def move_index_into_old_index_folder(baseindexpath): - command_move = "mv {} {}" - os.mkdir(join(baseindexpath, "old_index")) - for files in os.listdir(baseindexpath): - if not files == "old_index": - os.system(command_move.format(join(baseindexpath, files), join(join(baseindexpath, "old_index"), files))) + for cur_file in os.listdir(baseindexpath): + if not cur_file == "old_index": + shutil.move(join(baseindexpath, cur_file), join(join(baseindexpath, "old_index"), cur_file)) if __name__ == "__main__": diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 2d206b83..38ee4e83 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -112,9 +112,17 @@ function launching_queues { } function launching_scripts { + echo -e "\t* Checking configuration" + bash -c "./Update-conf.py" + exitStatus=$? + if [ $exitStatus -ge 1 ]; then + echo -e $RED"\t* Configuration not up-to-date"$DEFAULT + exit + fi + echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT + screen -dmS "Script" sleep 0.1 - echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x' @@ -257,7 +265,15 @@ for i in ${!options[@]}; do bash -c "./Shutdown.py" ;; Update-config) + echo -e "\t* Checking configuration" bash -c "./Update-conf.py" + exitStatus=$? + if [ $exitStatus -ge 1 ]; then + echo -e $RED"\t* Configuration not up-to-date"$DEFAULT + exit + else + echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT + fi ;; esac fi diff --git a/bin/Phone.py b/bin/Phone.py index 61000f98..cb32a691 100755 --- a/bin/Phone.py +++ b/bin/Phone.py @@ -13,6 +13,7 @@ It apply phone number regexes on paste content and warn if above a threshold. import time import re +import phonenumbers from packages import Paste from pubsublogger import publisher from Helper import Process @@ -35,6 +36,20 @@ def search_phone(message): p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste') #Send to duplicate p.populate_set_out(message, 'Duplicate') + stats = {} + for phone_number in results: + try: + x = phonenumbers.parse(phone_number, None) + country_code = x.country_code + if stats.get(country_code) is None: + stats[country_code] = 1 + else: + stats[country_code] = stats[country_code] + 1 + except: + pass + for country_code in stats: + if stats[country_code] > 4: + publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code)) if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) diff --git a/bin/Update-conf.py b/bin/Update-conf.py index 7ceada75..863ff436 100755 --- a/bin/Update-conf.py +++ b/bin/Update-conf.py @@ -5,11 +5,15 @@ import ConfigParser from ConfigParser import ConfigParser as cfgP import os from collections import OrderedDict +import sys +import shutil +#return true if the configuration is up-to-date def main(): configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + configfileBackup = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + '.backup' if not os.path.exists(configfile): raise Exception('Unable to find the configuration file. \ Did you set environment variables? \ @@ -45,8 +49,8 @@ def main(): dicoMissingItem[sec] = list_items if len(missingSection) == 0 and len(missingItem) == 0: - print("Configuration up-to-date") - return + #print("Configuration up-to-date") + return True print("/!\\ Configuration not complete. Missing following configuration: /!\\") print("+--------------------------------------------------------------------+") for section in missingSection: @@ -62,8 +66,12 @@ def main(): resp = raw_input("Do you want to auto fix it? [y/n] ") if resp != 'y': - return + return False else: + resp2 = raw_input("Do you want to keep a backup of the old configuration file? [y/n] ") + if resp2 == 'y': + shutil.move(configfile, configfileBackup) + #Do not keep item ordering in section. New items appened for section in missingItem: for item, value in dicoMissingItem[section]: @@ -75,6 +83,7 @@ def main(): with open(configfile, 'w') as f: cfg.write(f) + return True ''' Return a new dico with the section ordered as the old configuration with the updated one added ''' @@ -96,5 +105,8 @@ def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMis if __name__ == "__main__": - main() + if main(): + sys.exit() + else: + sys.exit(1) diff --git a/pip_packages_requirement.txt b/pip_packages_requirement.txt index ab5aa42c..8bffb7b5 100644 --- a/pip_packages_requirement.txt +++ b/pip_packages_requirement.txt @@ -26,6 +26,7 @@ ssdeep python-magic pybloomfiltermmap psutil +phonenumbers ipython flask