diff --git a/bin/BankAccount.py b/bin/BankAccount.py new file mode 100755 index 00000000..58fa3e64 --- /dev/null +++ b/bin/BankAccount.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +""" +The BankAccount Module +====================== + +It apply IBAN regexes on paste content and warn if above a threshold. + +""" + +import redis +import time +import re +import string +from itertools import chain + +from packages import Paste +from pubsublogger import publisher + +from Helper import Process + +import signal + +class TimeoutException(Exception): + pass + +def timeout_handler(signum, frame): + raise TimeoutException + +signal.signal(signal.SIGALRM, timeout_handler) + +_LETTERS_IBAN = chain(enumerate(string.digits + string.ascii_uppercase), + enumerate(string.ascii_lowercase, 10)) +LETTERS_IBAN = {ord(d): str(i) for i, d in _LETTERS_IBAN} + +def iban_number(iban): + return (iban[4:] + iban[:4]).translate(LETTERS_IBAN) + +def is_valid_iban(iban): + iban_numb = iban_number(iban) + iban_numb_check = iban_number(iban[:2] + '00' + iban[4:]) + check_digit = '{:0>2}'.format(98 - (int(iban_numb_check) % 97)) + if check_digit == iban[2:4] and int(iban_numb) % 97 == 1: + # valid iban + print('valid iban') + return True + return False + +def check_all_iban(l_iban, paste, filename): + nb_valid_iban = 0 + for iban in l_iban: + iban = iban[0]+iban[1]+iban[2] + iban = ''.join(e for e in iban if e.isalnum()) + #iban = iban.upper() + res = iban_regex_verify.findall(iban) + if res: + print('checking '+iban) + if is_valid_iban(iban): + print('------') + nb_valid_iban = nb_valid_iban + 1 + + if(nb_valid_iban > 0): + to_print = 'Iban;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name) + publisher.warning('{}Checked found {} IBAN;{}'.format( + to_print, nb_valid_iban, paste.p_path)) + msg = 'infoleak:automatic-detection="iban";{}'.format(filename) + p.populate_set_out(msg, 'Tags') + + #Send to duplicate + p.populate_set_out(filename, 'Duplicate') + +if __name__ == "__main__": + publisher.port = 6380 + publisher.channel = "Script" + + config_section = 'BankAccount' + + p = Process(config_section) + max_execution_time = p.config.getint("BankAccount", "max_execution_time") + + publisher.info("BankAccount started") + + message = p.get_from_set() + + #iban_regex = re.compile(r'\b[A-Za-z]{2}[0-9]{2}(?:[ ]?[0-9]{4}){4}(?:[ ]?[0-9]{1,2})?\b') + iban_regex = re.compile(r'\b([A-Za-z]{2}[ \-]?[0-9]{2})(?=(?:[ \-]?[A-Za-z0-9]){9,30})((?:[ \-]?[A-Za-z0-9]{3,5}){2,6})([ \-]?[A-Za-z0-9]{1,3})\b') + iban_regex_verify = re.compile(r'^([A-Z]{2})([0-9]{2})([A-Z0-9]{9,30})$') + + + while True: + + message = p.get_from_set() + + if message is not None: + + filename = message + paste = Paste.Paste(filename) + content = paste.get_p_content() + + signal.alarm(max_execution_time) + try: + l_iban = iban_regex.findall(content) + except TimeoutException: + print ("{0} processing timeout".format(paste.p_path)) + continue + else: + signal.alarm(0) + + if(len(l_iban) > 0): + check_all_iban(l_iban, paste, filename) + + else: + publisher.debug("Script BankAccount is Idling 10s") + time.sleep(10) diff --git a/bin/Credential.py b/bin/Credential.py index 5112f534..d1016586 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -79,8 +79,6 @@ if __name__ == "__main__": content = paste.get_p_content() creds = set(re.findall(regex_cred, content)) - publisher.warning('to_print') - if len(creds) == 0: continue diff --git a/bin/Helper.py b/bin/Helper.py index 7c035649..d90388f5 100755 --- a/bin/Helper.py +++ b/bin/Helper.py @@ -146,12 +146,15 @@ class Process(object): def populate_set_in(self): # monoproc src = self.modules.get(self.subscriber_name, 'subscribe') - self.pubsub.setup_subscribe(src) - for msg in self.pubsub.subscribe(): - in_set = self.subscriber_name + 'in' - self.r_temp.sadd(in_set, msg) - self.r_temp.hset('queues', self.subscriber_name, - int(self.r_temp.scard(in_set))) + if src != 'Redis': + self.pubsub.setup_subscribe(src) + for msg in self.pubsub.subscribe(): + in_set = self.subscriber_name + 'in' + self.r_temp.sadd(in_set, msg) + self.r_temp.hset('queues', self.subscriber_name, + int(self.r_temp.scard(in_set))) + else: + print('{} has no suscriber'.format(self.subscriber_name)) def get_from_set(self): # multiproc diff --git a/bin/Keys.py b/bin/Keys.py index 7b1ec7dc..d3c292ba 100755 --- a/bin/Keys.py +++ b/bin/Keys.py @@ -71,6 +71,14 @@ def search_key(paste): p.populate_set_out(msg, 'Tags') find = True + if '---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----' in content: + publisher.warning('{} has an ssh2 private key message'.format(paste.p_name)) + print('SSH2 private key message found') + + msg = 'infoleak:automatic-detection="private-ssh-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') + find = True + if '-----BEGIN OpenVPN Static key V1-----' in content: publisher.warning('{} has an openssh private key message'.format(paste.p_name)) print('OpenVPN Static key message found') diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 3acd7bb2..c3bfd8cf 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -144,6 +144,8 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x' sleep 0.1 + screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x' + sleep 0.1 screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x' sleep 0.1 screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x' diff --git a/bin/Mixer.py b/bin/Mixer.py index 98709ea5..96f20815 100755 --- a/bin/Mixer.py +++ b/bin/Mixer.py @@ -68,6 +68,12 @@ if __name__ == '__main__': db=cfg.getint("Redis_Mixer_Cache", "db"), decode_responses=True) + server_cache = redis.StrictRedis( + host=cfg.get("Redis_Log_submit", "host"), + port=cfg.getint("Redis_Log_submit", "port"), + db=cfg.getint("Redis_Log_submit", "db"), + decode_responses=True) + # LOGGING # publisher.info("Feed Script started to receive & publish.") @@ -184,7 +190,17 @@ if __name__ == '__main__': publisher.debug("Empty Paste: {0} not processed".format(message)) else: print("Empty Queues: Waiting...") + if int(time.time() - time_1) > refresh_time: + # update internal feeder + list_feeder = server_cache.hkeys("mixer_cache:list_feeder") + if list_feeder: + for feeder in list_feeder: + count = int(server_cache.hget("mixer_cache:list_feeder", feeder)) + if count is None: + count = 0 + processed_paste_per_feeder[feeder] = processed_paste_per_feeder.get(feeder, 0) + count + processed_paste = processed_paste + count print(processed_paste_per_feeder) to_print = 'Mixer; ; ; ;mixer_all All_feeders Processed {0} paste(s) in {1}sec'.format(processed_paste, refresh_time) print(to_print) @@ -204,5 +220,8 @@ if __name__ == '__main__': duplicated_paste_per_feeder[feeder] = 0 time_1 = time.time() + + # delete internal feeder list + server_cache.delete("mixer_cache:list_feeder") time.sleep(0.5) continue diff --git a/bin/feeder/pystemon-feeder.py b/bin/feeder/pystemon-feeder.py index 50ffaeba..a59a0a5b 100755 --- a/bin/feeder/pystemon-feeder.py +++ b/bin/feeder/pystemon-feeder.py @@ -62,12 +62,13 @@ while True: print(paste) if paste is None: continue - socket.send("%d %s" % (topic, paste)) + socket.send_string("%d %s" % (topic, paste)) topic = 102 try: - messagedata = open(pystemonpath+paste).read() - socket.send("%d %s %s" % (topic, paste, base64.b64encode(messagedata))) - sleep_inc = sleep_inc-0.01 if sleep_inc-0.01 > 0 else 0 + with open(pystemonpath+paste, 'rb') as f: #.read() + messagedata = f.read() + socket.send_string("%d %s %s" % (topic, paste, base64.b64encode(messagedata).decode())) + sleep_inc = sleep_inc-0.01 if sleep_inc-0.01 > 0 else 0 except IOError as e: # file not found, could be a buffering issue -> increase sleeping time print('IOError: Increasing sleep time') diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index b10d4af0..eead0357 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -32,6 +32,8 @@ sender_port = 1337 ##### Flask ##### [Flask] +#Number of logs to display in the dashboard +max_dashboard_logs = 15 #Maximum number of character to display in the toolip max_preview_char = 250 #Maximum number of character to display in the modal @@ -44,6 +46,9 @@ minute_processed_paste = 10 DiffMaxLineLength = 10000 #### Modules #### +[BankAccount] +max_execution_time = 60 + [Categ] #Minimum number of match between the paste and the category file matchingThreshold=1 diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 712f5bab..8cd8f570 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -51,6 +51,10 @@ publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Re subscribe = Redis_CreditCards publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags +[BankAccount] +subscribe = Redis_Global +publish = Redis_Duplicate,Redis_Tags + [Mail] subscribe = Redis_Mail publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags @@ -130,4 +134,5 @@ subscribe = Redis_Global publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags [submit_paste] +subscribe = Redis publish = Redis_Mixer diff --git a/bin/submit_paste.py b/bin/submit_paste.py index 066fb846..a999ec39 100755 --- a/bin/submit_paste.py +++ b/bin/submit_paste.py @@ -40,6 +40,9 @@ def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name): relay_message = "{0} {1}".format(save_path, gzip64encoded) p.populate_set_out(relay_message, 'Mixer') + # increase nb of paste by feeder name + r_serv_log_submit.hincrby("mixer_cache:list_feeder", "submitted", 1) + # add tags add_tags(ltags, ltagsgalaxies, full_path) diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index 43c65060..256ea3a8 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -144,7 +144,9 @@ bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted') - # VT +max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs")) + +# VT try: from virusTotalKEYS import vt_key if vt_key != '': diff --git a/var/www/modules/PasteSubmit/templates/PasteSubmit.html b/var/www/modules/PasteSubmit/templates/PasteSubmit.html index 25e19c50..a0636332 100644 --- a/var/www/modules/PasteSubmit/templates/PasteSubmit.html +++ b/var/www/modules/PasteSubmit/templates/PasteSubmit.html @@ -5,7 +5,8 @@ - Analysis Information Leak framework Dashboard + Submit Paste - AIL + diff --git a/var/www/modules/PasteSubmit/templates/edit_tag_export.html b/var/www/modules/PasteSubmit/templates/edit_tag_export.html index 01822042..d9e8a471 100644 --- a/var/www/modules/PasteSubmit/templates/edit_tag_export.html +++ b/var/www/modules/PasteSubmit/templates/edit_tag_export.html @@ -5,7 +5,8 @@ - Analysis Information Leak framework + Tags Export - AIL + diff --git a/var/www/modules/PasteSubmit/templates/header_PasteSubmit.html b/var/www/modules/PasteSubmit/templates/header_PasteSubmit.html index 9abd9029..4a6a3b77 100644 --- a/var/www/modules/PasteSubmit/templates/header_PasteSubmit.html +++ b/var/www/modules/PasteSubmit/templates/header_PasteSubmit.html @@ -1 +1 @@ -
  • PasteSubmit
    • +
  • Submit Paste
    • diff --git a/var/www/modules/PasteSubmit/templates/submiting.html b/var/www/modules/PasteSubmit/templates/submiting.html index b7ad78e3..6174742f 100644 --- a/var/www/modules/PasteSubmit/templates/submiting.html +++ b/var/www/modules/PasteSubmit/templates/submiting.html @@ -5,7 +5,8 @@ - Analysis Information Leak framework Dashboard + Submit Paste - AIL + diff --git a/var/www/modules/Tags/templates/Tags.html b/var/www/modules/Tags/templates/Tags.html index c9ecb06d..143fddb3 100644 --- a/var/www/modules/Tags/templates/Tags.html +++ b/var/www/modules/Tags/templates/Tags.html @@ -5,7 +5,8 @@ - Analysis Information Leak framework Dashboard + Tags - AIL + diff --git a/var/www/modules/Tags/templates/edit_galaxy.html b/var/www/modules/Tags/templates/edit_galaxy.html index c6e10f6c..0d1b1bf9 100644 --- a/var/www/modules/Tags/templates/edit_galaxy.html +++ b/var/www/modules/Tags/templates/edit_galaxy.html @@ -5,7 +5,8 @@ - Analysis Information Leak framework Dashboard + Edit Galaxy - AIL + diff --git a/var/www/modules/Tags/templates/edit_taxonomie.html b/var/www/modules/Tags/templates/edit_taxonomie.html index 74ea5b9c..45f44282 100644 --- a/var/www/modules/Tags/templates/edit_taxonomie.html +++ b/var/www/modules/Tags/templates/edit_taxonomie.html @@ -5,7 +5,8 @@ - Analysis Information Leak framework Dashboard + Edit Taxonomie - AIL + diff --git a/var/www/modules/Tags/templates/galaxies.html b/var/www/modules/Tags/templates/galaxies.html index 5013c356..2dc6d473 100644 --- a/var/www/modules/Tags/templates/galaxies.html +++ b/var/www/modules/Tags/templates/galaxies.html @@ -5,7 +5,8 @@ - Analysis Information Leak framework Dashboard + Galaxies - AIL + diff --git a/var/www/modules/Tags/templates/tag_galaxy_info.html b/var/www/modules/Tags/templates/tag_galaxy_info.html index a1544fcd..9ee05049 100644 --- a/var/www/modules/Tags/templates/tag_galaxy_info.html +++ b/var/www/modules/Tags/templates/tag_galaxy_info.html @@ -5,7 +5,8 @@ - Analysis Information Leak framework Dashboard + Galaxy Tag Info - AIL + diff --git a/var/www/modules/Tags/templates/tagged.html b/var/www/modules/Tags/templates/tagged.html index 1053041f..c0628eb7 100644 --- a/var/www/modules/Tags/templates/tagged.html +++ b/var/www/modules/Tags/templates/tagged.html @@ -5,7 +5,8 @@ - Analysis Information Leak framework Dashboard + Tags - AIL + diff --git a/var/www/modules/Tags/templates/taxonomies.html b/var/www/modules/Tags/templates/taxonomies.html index 74feda6f..83d8a930 100644 --- a/var/www/modules/Tags/templates/taxonomies.html +++ b/var/www/modules/Tags/templates/taxonomies.html @@ -5,7 +5,8 @@ - Analysis Information Leak framework Dashboard + Taxonomies - AIL + diff --git a/var/www/modules/browsepastes/templates/browse_important_paste.html b/var/www/modules/browsepastes/templates/browse_important_paste.html index faa7ed3d..a68e0f48 100644 --- a/var/www/modules/browsepastes/templates/browse_important_paste.html +++ b/var/www/modules/browsepastes/templates/browse_important_paste.html @@ -5,7 +5,8 @@ - Analysis Information Leak framework Dashboard + Browse Important Paste - AIL + diff --git a/var/www/modules/dashboard/Flask_dashboard.py b/var/www/modules/dashboard/Flask_dashboard.py index 563eb007..6d7992d7 100644 --- a/var/www/modules/dashboard/Flask_dashboard.py +++ b/var/www/modules/dashboard/Flask_dashboard.py @@ -5,10 +5,14 @@ Flask functions and routes for the dashboard page ''' import json - +import os import datetime +import time import flask -from flask import Flask, render_template, jsonify, request, Blueprint + +from Date import Date + +from flask import Flask, render_template, jsonify, request, Blueprint, url_for # ============ VARIABLES ============ import Flask_config @@ -18,6 +22,8 @@ cfg = Flask_config.cfg r_serv = Flask_config.r_serv r_serv_log = Flask_config.r_serv_log +max_dashboard_logs = Flask_config.max_dashboard_logs + dashboard = Blueprint('dashboard', __name__, template_folder='templates') # ============ FUNCTIONS ============ @@ -62,12 +68,87 @@ def get_queues(r): return newData +def get_date_range(date_from, num_day): + date = Date(str(date_from[0:4])+str(date_from[4:6]).zfill(2)+str(date_from[6:8]).zfill(2)) + date_list = [] + + for i in range(0, num_day+1): + new_date = date.substract_day(i) + date_list.append(new_date[0:4] +'-'+ new_date[4:6] +'-'+ new_date[6:8]) + + return date_list + +def dashboard_alert(log): + # check if we need to display this log + if len(log)>50: + date = log[1:5]+log[6:8]+log[9:11] + utc_str = log[1:20] + log = log[46:].split(';') + if len(log) == 6: + time = datetime_from_utc_to_local(utc_str) + path = url_for('showsavedpastes.showsavedpaste',paste=log[5]) + + res = {'date': date, 'time': time, 'script': log[0], 'domain': log[1], 'date_paste': log[2], + 'paste': log[3], 'message': log[4], 'path': path} + return res + else: + return False + else: + return False + +def datetime_from_utc_to_local(utc_str): + utc_datetime = datetime.datetime.strptime(utc_str, '%Y-%m-%d %H:%M:%S') + now_timestamp = time.time() + offset = datetime.datetime.fromtimestamp(now_timestamp) - datetime.datetime.utcfromtimestamp(now_timestamp) + local_time_str = (utc_datetime + offset).strftime('%H:%M:%S') + return local_time_str + # ============ ROUTES ============ @dashboard.route("/_logs") def logs(): return flask.Response(event_stream(), mimetype="text/event-stream") +@dashboard.route("/_get_last_logs_json") +def get_last_logs_json(): + date = datetime.datetime.now().strftime("%Y%m%d") + + max_day_search = 6 + day_search = 0 + warning_found = 0 + warning_to_found = max_dashboard_logs + + last_logs = [] + + date_range = get_date_range(date, max_day_search) + while max_day_search != day_search and warning_found != warning_to_found: + + filename_warning_log = 'logs/Script_warn-'+ date_range[day_search] +'.log' + filename_log = os.path.join(os.environ['AIL_HOME'], filename_warning_log) + + try: + with open(filename_log, 'r') as f: + lines = f.read().splitlines() + curr_index = -1 + while warning_found != warning_to_found: + try: + # get lasts warning logs + log_warn = dashboard_alert(lines[curr_index]) + if log_warn != False: + last_logs.append(log_warn) + warning_found = warning_found + 1 + curr_index = curr_index - 1 + + except IndexError: + # check previous warning log file + day_search = day_search + 1 + break + except FileNotFoundError: + # check previous warning log file + day_search = day_search + 1 + + return jsonify(list(reversed(last_logs))) + @dashboard.route("/_stuff", methods=['GET']) def stuff(): @@ -78,7 +159,12 @@ def stuff(): def index(): default_minute = cfg.get("Flask", "minute_processed_paste") threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module") - return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module) + log_select = {10, 25, 50, 100} + log_select.add(max_dashboard_logs) + log_select = list(log_select) + log_select.sort() + return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module, + log_select=log_select, selected=max_dashboard_logs) # ========= REGISTRATION ========= app.register_blueprint(dashboard) diff --git a/var/www/modules/dashboard/templates/index.html b/var/www/modules/dashboard/templates/index.html index e5d61014..e7331b6f 100644 --- a/var/www/modules/dashboard/templates/index.html +++ b/var/www/modules/dashboard/templates/index.html @@ -6,6 +6,7 @@ Analysis Information Leak framework Dashboard + @@ -31,6 +32,14 @@ }; update_values(); + @@ -136,10 +145,13 @@
    INFO @@ -182,6 +194,66 @@ - + - +