diff --git a/Dockerfile b/Dockerfile index 71318ba4..340e5014 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,19 @@ FROM ubuntu:16.04 -RUN mkdir /opt/AIL && apt-get update -y \ - && apt-get install git python-dev build-essential \ - libffi-dev libssl-dev libfuzzy-dev wget sudo -y +# Make sure that all updates are in place +RUN apt-get clean && apt-get update -y && apt-get upgrade -y \ + && apt-get dist-upgrade -y && apt-get autoremove -y + +# Install needed packages +RUN apt-get install git python-dev build-essential \ + libffi-dev libssl-dev libfuzzy-dev wget sudo -y # Adding sudo command RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo RUN echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers # Installing AIL dependencies +RUN mkdir /opt/AIL ADD . /opt/AIL WORKDIR /opt/AIL RUN ./installing_deps.sh diff --git a/HOWTO.md b/HOWTO.md index 2228d3a6..c21a970f 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -27,7 +27,7 @@ Feed data to AIL: 4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly -5. Launch pystemon-feeder ``` ./pystemon-feeder.py ``` +5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ``` How to create a new module diff --git a/README.md b/README.md index 72166c58..e8af8e5f 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,6 @@ Type these command lines for a fully automated installation and start AIL framew git clone https://github.com/CIRCL/AIL-framework.git cd AIL-framework ./installing_deps.sh -cd var/www/ -./update_thirdparty.sh cd ~/AIL-framework/ . ./AILENV/bin/activate cd bin/ @@ -155,6 +153,11 @@ Eventually you can browse the status of the AIL framework website at the followi http://localhost:7000/ ``` +Training +-------- + +CIRCL organises training on how to use or extend the AIL framework. The next training will be [Thursday, 20 Dec](https://en.xing-events.com/ZEQWMLJ.html) in Luxembourg. + HOWTO ----- diff --git a/bin/Crawler.py b/bin/Crawler.py index 99917c49..0f69cfe6 100755 --- a/bin/Crawler.py +++ b/bin/Crawler.py @@ -27,17 +27,27 @@ def crawl_onion(url, domain, date, date_month, message): if super_father is None: super_father=paste - try: - r = requests.get(splash_url , timeout=30.0) - except Exception: - # TODO: relaunch docker or send error message + retry = True + nb_retry = 0 + while retry: + try: + r = requests.get(splash_url , timeout=30.0) + retry = False + except Exception: + # TODO: relaunch docker or send error message + nb_retry += 1 - on_error_send_message_back_in_queue(type_hidden_service, domain, message) - publisher.error('{} SPASH DOWN'.format(splash_url)) - print('--------------------------------------') - print(' \033[91m DOCKER SPLASH DOWN\033[0m') - print(' {} DOWN'.format(splash_url)) - exit(1) + if nb_retry == 30: + on_error_send_message_back_in_queue(type_hidden_service, domain, message) + publisher.error('{} SPASH DOWN'.format(splash_url)) + print('--------------------------------------') + print(' \033[91m DOCKER SPLASH DOWN\033[0m') + print(' {} DOWN'.format(splash_url)) + exit(1) + + print(' \033[91m DOCKER SPLASH NOT AVAILABLE\033[0m') + print(' Retry({}) in 10 seconds'.format(nb_retry)) + time.sleep(10) if r.status_code == 200: process = subprocess.Popen(["python", './torcrawler/tor_crawler.py', splash_url, type_hidden_service, url, domain, paste, super_father], diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 684af83b..549c0425 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -9,12 +9,30 @@ WHITE="\\033[0;02m" YELLOW="\\033[1;33m" CYAN="\\033[1;36m" -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; +# Getting CWD where bash script resides +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd |sed 's/bin//' )" +export AIL_HOME="${DIR}" +cd ${AIL_HOME} + +if [ -e "${DIR}/AILENV/bin/python" ]; then + echo "AIL-framework virtualenv seems to exist, good" + ENV_PY="${DIR}/AILENV/bin/python" +else + echo "Please make sure you have a AIL-framework environment, au revoir" + exit 1 +fi + +# redis-server is bundled during install +## [ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1 + +export AIL_BIN=${AIL_HOME}/bin/ +export AIL_FLASK=${AIL_HOME}/var/www/ +export AIL_REDIS=${AIL_HOME}/redis/src/ +export AIL_ARDB=${AIL_HOME}/ardb/src/ +export AIL_VENV=${AIL_HOME}/AILENV/ + +export PATH=$AIL_VENV/bin:$PATH export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH export PATH=$AIL_ARDB:$PATH @@ -91,9 +109,9 @@ function launching_logs { screen -dmS "Logging_AIL" sleep 0.1 echo -e $GREEN"\t* Launching logging process"$DEFAULT - screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Queuing -l ../logs/; read x' + screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/; read x" sleep 0.1 - screen -S "Logging_AIL" -X screen -t "LogScript" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Script -l ../logs/; read x' + screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/; read x" } function launching_queues { @@ -101,16 +119,16 @@ function launching_queues { sleep 0.1 echo -e $GREEN"\t* Launching all the queues"$DEFAULT - screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x' + screen -S "Queue_AIL" -X screen -t "Queues" bash -c "cd ${AIL_BIN}; ${ENV_PY} launch_queues.py; read x" } function checking_configuration { bin_dir=${AIL_HOME}/bin echo -e "\t* Checking configuration" if [ "$1" == "automatic" ]; then - bash -c "python3 $bin_dir/Update-conf.py True" + bash -c "${ENV_PY} $bin_dir/Update-conf.py True" else - bash -c "python3 $bin_dir/Update-conf.py False" + bash -c "${ENV_PY} $bin_dir/Update-conf.py False" fi exitStatus=$? @@ -128,75 +146,75 @@ function launching_scripts { sleep 0.1 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT - screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x' + screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x' + screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x' + screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x' + screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x' + screen -S "Script_AIL" -X screen -t "Lines" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Lines.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x' + screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x' + screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x' + screen -S "Script_AIL" -X screen -t "Tokenize" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tokenize.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x' + screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x' + screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x' + screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x' + screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x' + screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x' + screen -S "Script_AIL" -X screen -t "Web" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Web.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x' + screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x' + screen -S "Script_AIL" -X screen -t "Curve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Curve.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x' + screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CurveManageTopSets.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexForTermsFrequency.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SetForTermsFrequency.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x' + screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x' + screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x' + screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x' + screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Bitcoin.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x' + screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x' + screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x' + screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x' + screen -S "Script_AIL" -X screen -t "WebStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./WebStats.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x' + screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x' + screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SQLInjectionDetection.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x' + screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./LibInjection.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x' + screen -S "Script_AIL" -X screen -t "alertHandler" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./alertHandler.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x' + screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x' + screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x' + screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x' + screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x" } @@ -219,7 +237,7 @@ function launching_crawler { sleep 0.1 for ((i=first_port;i<=last_port;i++)); do - screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ./Crawler.py onion '$i'; read x' + screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py onion $i; read x" sleep 0.1 done @@ -342,7 +360,7 @@ function launch_flask { screen -dmS "Flask_AIL" sleep 0.1 echo -e $GREEN"\t* Launching Flask server"$DEFAULT - screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x" + screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ${ENV_PY} ./Flask_server.py; read x" else echo -e $RED"\t* A Flask screen is already launched"$DEFAULT fi @@ -353,9 +371,9 @@ function launch_feeder { screen -dmS "Feeder_Pystemon" sleep 0.1 echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT - screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ./feeder/pystemon-feeder.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./feeder/pystemon-feeder.py; read x" sleep 0.1 - screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; python2 pystemon.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c "cd ${AIL_HOME}/../pystemon; ${ENV_PY} ./pystemon.py; read x" else echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT fi diff --git a/bin/feeder/pystemon-feeder.py b/bin/feeder/pystemon-feeder.py index b6680ee9..280849ba 100755 --- a/bin/feeder/pystemon-feeder.py +++ b/bin/feeder/pystemon-feeder.py @@ -61,10 +61,10 @@ topic = '102' while True: time.sleep(base_sleeptime + sleep_inc) paste = r.lpop("pastes") - print(paste) if paste is None: continue try: + print(paste) with open(pystemonpath+paste, 'rb') as f: #.read() messagedata = f.read() path_to_send = pastes_directory+paste diff --git a/bin/feeder/test-zmq.py b/bin/feeder/test-zmq.py index f6f28aa1..110c5de2 100644 --- a/bin/feeder/test-zmq.py +++ b/bin/feeder/test-zmq.py @@ -20,7 +20,7 @@ socket.connect ("tcp://crf.circl.lu:%s" % port) # 102 Full pastes in raw base64(gz) topicfilter = "102" -socket.setsockopt(zmq.SUBSCRIBE, topicfilter) +socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter) while True: message = socket.recv() diff --git a/bin/packages/HiddenServices.py b/bin/packages/HiddenServices.py index 170e1dc3..5db49754 100755 --- a/bin/packages/HiddenServices.py +++ b/bin/packages/HiddenServices.py @@ -81,8 +81,12 @@ class HiddenServices(object): return '' return origin_paste.replace(self.paste_directory+'/', '') - def get_domain_tags(self): - return self.tags + def get_domain_tags(self, update=False): + if not update: + return self.tags + else: + self.get_last_crawled_pastes() + return self.tags def update_domain_tags(self, children): p_tags = self.r_serv_metadata.smembers('tag:'+children) diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 6efb4be4..b5980766 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -206,6 +206,9 @@ dns = 8.8.8.8 [Mail] dns = 8.8.8.8 +[Web] +dns = 149.13.33.69 + # Indexer configuration [Indexer] type = whoosh diff --git a/bin/packages/lib_refine.py b/bin/packages/lib_refine.py index 5d2af0a9..32f56900 100644 --- a/bin/packages/lib_refine.py +++ b/bin/packages/lib_refine.py @@ -1,6 +1,8 @@ #!/usr/bin/python3 import re +import os +import configparser import dns.resolver from pubsublogger import publisher @@ -101,11 +103,20 @@ def checking_MX_record(r_serv, adress_set, addr_dns): def checking_A_record(r_serv, domains_set): + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + cfg = configparser.ConfigParser() + cfg.read(configfile) + dns_server = cfg.get("Web", "dns") + score = 0 num = len(domains_set) WalidA = set([]) resolver = dns.resolver.Resolver() - resolver.nameservers = ['149.13.33.69'] + resolver.nameservers = [dns_server] resolver.timeout = 5 resolver.lifetime = 2 diff --git a/bin/torcrawler/launch_splash_crawler.sh b/bin/torcrawler/launch_splash_crawler.sh index 412022c1..e47efc36 100755 --- a/bin/torcrawler/launch_splash_crawler.sh +++ b/bin/torcrawler/launch_splash_crawler.sh @@ -32,6 +32,21 @@ if [ -z "${p}" ] || [ -z "${f}" ] || [ -z "${n}" ]; then usage; fi +RED="\\033[1;31m" +DEFAULT="\\033[0;39m" +GREEN="\\033[1;32m" +WHITE="\\033[0;02m" + +if [ ! -d "${f}" ]; then + printf "$RED\n Error -f, proxy-profiles directory: $WHITE${f}$RED not found\n$DEFAULT Please check if you enter the correct path\n" + exit 1 +fi + +if [ ! -f "${f}default.ini" ]; then + printf "$RED\n Error -f, proxy configuration file:$WHITE default.ini$RED not found\n$DEFAULT Please check if you enter the correct path\n" + exit 1 +fi + screen -dmS "Docker_Splash" sleep 0.1 @@ -39,5 +54,5 @@ for ((i=0;i<=$((${n} - 1));i++)); do port_number=$((${p} + $i)) screen -S "Docker_Splash" -X screen -t "docker_splash:$port_number" bash -c 'sudo docker run -p '$port_number':8050 --cpus=1 --memory=4.5G -v '$f':/etc/splash/proxy-profiles/ --net="bridge" scrapinghub/splash; read x' sleep 0.1 - echo " Splash server launched on port $port_number" + printf "$GREEN Splash server launched on port $port_number$DEFAULT\n" done diff --git a/installing_deps.sh b/installing_deps.sh index 6110d534..484ca770 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -5,7 +5,7 @@ set -x sudo apt-get update -sudo apt-get install python3-pip python-virtualenv python3-dev python3-tk libfreetype6-dev \ +sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6-dev \ screen g++ python-tk unzip libsnappy-dev cmake -y #optional tor install diff --git a/pip3_packages_requirement.txt b/pip3_packages_requirement.txt index dd447d5c..3991e158 100644 --- a/pip3_packages_requirement.txt +++ b/pip3_packages_requirement.txt @@ -2,7 +2,7 @@ pymisp thehive4py -redis +redis==2.10.6 #filemagic conflict with magic crcmod mmh3 @@ -13,7 +13,6 @@ zmq langid #Essential -redis pyzmq dnspython logbook diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index ee5d7ee1..3cc19e55 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -8,7 +8,7 @@ import redis import datetime import sys import os -from flask import Flask, render_template, jsonify, request, Blueprint +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for from Date import Date from HiddenServices import HiddenServices @@ -39,6 +39,16 @@ def get_date_range(num_day): return list(reversed(date_list)) +def substract_date(date_from, date_to): + date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) + date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) + delta = date_to - date_from # timedelta + l_date = [] + for i in range(delta.days + 1): + date = date_from + datetime.timedelta(i) + l_date.append( date.strftime('%Y%m%d') ) + return l_date + def unpack_paste_tags(p_tags): l_tags = [] for tag in p_tags: @@ -93,6 +103,94 @@ def hiddenServices_page(): return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains) +@hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST']) +def get_onions_by_daterange(): + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + domains_up = request.form.get('domains_up') + domains_down = request.form.get('domains_down') + domains_tags = request.form.get('domains_tags') + + return redirect(url_for('hiddenServices.show_domains_by_daterange', date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, domains_tags=domains_tags)) + +@hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET']) +def show_domains_by_daterange(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + domains_up = request.args.get('domains_up') + domains_down = request.args.get('domains_down') + domains_tags = request.args.get('domains_tags') + + date_range = [] + if date_from is not None and date_to is not None: + #change format + try: + if len(date_from) != 8: + date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] + date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] + date_range = substract_date(date_from, date_to) + except: + pass + + if not date_range: + date_range.append(datetime.date.today().strftime("%Y%m%d")) + date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8] + date_to = date_from + + else: + date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8] + date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8] + + domains_by_day = {} + domain_metadata = {} + for date in date_range: + if domains_up: + domains_up = True + domains_by_day[date] = list(r_serv_onion.smembers('onion_up:{}'.format(date))) + for domain in domains_by_day[date]: + h = HiddenServices(domain, 'onion') + domain_metadata[domain] = {} + if domains_tags: + domains_tags = True + domain_metadata[domain]['tags'] = h.get_domain_tags(update=True) + + domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check') + if domain_metadata[domain]['last_check'] is None: + domain_metadata[domain]['last_check'] = '********' + domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen') + if domain_metadata[domain]['first_seen'] is None: + domain_metadata[domain]['first_seen'] = '********' + domain_metadata[domain]['status_text'] = 'UP' + domain_metadata[domain]['status_color'] = 'Green' + domain_metadata[domain]['status_icon'] = 'fa-check-circle' + + if domains_down: + domains_down = True + domains_by_day_down = list(r_serv_onion.smembers('onion_down:{}'.format(date))) + if domains_up: + domains_by_day[date].extend(domains_by_day_down) + else: + domains_by_day[date] = domains_by_day_down + for domain in domains_by_day_down: + #h = HiddenServices(onion_domain, 'onion') + domain_metadata[domain] = {} + #domain_metadata[domain]['tags'] = h.get_domain_tags() + + domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check') + if domain_metadata[domain]['last_check'] is None: + domain_metadata[domain]['last_check'] = '********' + domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen') + if domain_metadata[domain]['first_seen'] is None: + domain_metadata[domain]['first_seen'] = '********' + + domain_metadata[domain]['status_text'] = 'DOWN' + domain_metadata[domain]['status_color'] = 'Red' + domain_metadata[domain]['status_icon'] = 'fa-times-circle' + + return render_template("domains.html", date_range=date_range, domains_by_day=domains_by_day, domain_metadata=domain_metadata, + date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, + domains_tags=domains_tags, bootstrap_label=bootstrap_label) + @hiddenServices.route("/hiddenServices/onion_domain", methods=['GET']) def onion_domain(): onion_domain = request.args.get('onion_domain') diff --git a/var/www/modules/hiddenServices/templates/domains.html b/var/www/modules/hiddenServices/templates/domains.html new file mode 100644 index 00000000..136291b1 --- /dev/null +++ b/var/www/modules/hiddenServices/templates/domains.html @@ -0,0 +1,291 @@ + + + + + + + + Hidden Service - AIL + + + + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+ +
+
+ + {% for date in date_range %} + {% if domains_by_day[date]%} +
+
+

{{'{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])}}

+
+
+ + + + + + + + + + + + {% for domain in domains_by_day[date] %} + + + + + + + {% endfor %} + +
DomainFirst SeenLast CheckStatus
+ {{ domain }} +
+ {% for tag in domain_metadata[domain]['tags'] %} + + {{ tag }} {{ domain_metadata[domain]['tags'][tag] }} + + {% endfor %} +
+ 		{{'{}/{}/{}'.format(domain_metadata[domain]['first_seen'][0:4], domain_metadata[domain]['first_seen'][4:6], domain_metadata[domain]['first_seen'][6:8])}}{{'{}/{}/{}'.format(domain_metadata[domain]['last_check'][0:4], domain_metadata[domain]['last_check'][4:6], domain_metadata[domain]['last_check'][6:8])}}
+ + {{domain_metadata[domain]['status_text']}} +
+
+
+
+
+ {% endif %} + {% endfor %} + + +
+ +
+
+
+ Select domains by date range : +
+
+
+
+
+
+ + +
+
+ + +
+
+ +
+
+
+
+ +
+
+ +
+
+
+ + +
+
+ +
+ +
+ +
+ + + + + + + + diff --git a/var/www/modules/hiddenServices/templates/hiddenServices.html b/var/www/modules/hiddenServices/templates/hiddenServices.html index 59aeb2ae..b781a26a 100644 --- a/var/www/modules/hiddenServices/templates/hiddenServices.html +++ b/var/www/modules/hiddenServices/templates/hiddenServices.html @@ -12,11 +12,17 @@ + + +