From b1b7d33c9657489f14d55ac4ef0a8a42bd99f308 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 16 May 2018 14:39:01 +0200 Subject: [PATCH] tags --- bin/ApiKey.py | 7 + bin/Base64.py | 3 + bin/Bitcoin.py | 7 + bin/Credential.py | 3 + bin/CreditCards.py | 3 + bin/Cve.py | 3 + bin/Keys.py | 29 ++ bin/LAUNCH.sh | 6 + bin/Mail.py | 3 + bin/Onion.py | 3 + bin/Phone.py | 4 + bin/SQLInjectionDetection.py | 3 + bin/Tags.py | 68 +++++ var/www/modules/Tags/Flask_Tags.py | 150 +++++++++ var/www/modules/Tags/templates/Tags.html | 78 +++++ .../modules/Tags/templates/header_Tags.html | 1 + var/www/modules/Tags/templates/tagged.html | 285 ++++++++++++++++++ var/www/modules/showpaste/Flask_showpaste.py | 12 +- .../showpaste/templates/show_saved_paste.html | 8 +- 19 files changed, 674 insertions(+), 2 deletions(-) create mode 100755 bin/Tags.py create mode 100644 var/www/modules/Tags/Flask_Tags.py create mode 100644 var/www/modules/Tags/templates/Tags.html create mode 100644 var/www/modules/Tags/templates/header_Tags.html create mode 100644 var/www/modules/Tags/templates/tagged.html diff --git a/bin/ApiKey.py b/bin/ApiKey.py index 8ce7e2b4..e7ded9b2 100755 --- a/bin/ApiKey.py +++ b/bin/ApiKey.py @@ -41,6 +41,8 @@ def search_api_key(message): print(to_print) publisher.warning('{}Checked {} found Google API Key;{}'.format( to_print, len(google_api_key), paste.p_path)) + msg = 'infoleak:automatic-detection="google-api-key";{}'.format(filename) + p.populate_set_out(msg, 'Tags') if(len(aws_access_key) > 0 or len(aws_secret_key) > 0): print('found AWS key') @@ -48,8 +50,13 @@ def search_api_key(message): total = len(aws_access_key) + len(aws_secret_key) publisher.warning('{}Checked {} found AWS Key;{}'.format( to_print, total, paste.p_path)) + msg = 'infoleak:automatic-detection="aws-key";{}'.format(filename) + p.populate_set_out(msg, 'Tags') + msg = 'infoleak:automatic-detection="api-key";{}'.format(filename) + p.populate_set_out(msg, 'Tags') + msg = 'apikey;{}'.format(filename) p.populate_set_out(msg, 'alertHandler') #Send to duplicate diff --git a/bin/Base64.py b/bin/Base64.py index c7700994..960ca6de 100755 --- a/bin/Base64.py +++ b/bin/Base64.py @@ -65,6 +65,9 @@ def search_base64(content, message): msg = ('base64;{}'.format(message)) p.populate_set_out( msg, 'alertHandler') + msg = 'infoleak:automatic-detection="base64";{}'.format(message) + p.populate_set_out(msg, 'Tags') + def save_base64_as_file(decode, type, hash, json_data): filename_b64 = os.path.join(os.environ['AIL_HOME'], diff --git a/bin/Bitcoin.py b/bin/Bitcoin.py index 42468759..5ec2199f 100755 --- a/bin/Bitcoin.py +++ b/bin/Bitcoin.py @@ -63,7 +63,14 @@ def search_key(content, message, paste): publisher.warning(to_print) msg = ('bitcoin;{}'.format(message)) p.populate_set_out( msg, 'alertHandler') + + msg = 'infoleak:automatic-detection="bitcoin-address";{}'.format(message) + p.populate_set_out(msg, 'Tags') + if(key): + msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') + to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name) publisher.warning('{}Detected {} Bitcoin private key;{}'.format( diff --git a/bin/Credential.py b/bin/Credential.py index fde80d12..5112f534 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -105,6 +105,9 @@ if __name__ == "__main__": msg = 'credential;{}'.format(filepath) p.populate_set_out(msg, 'alertHandler') + msg = 'infoleak:automatic-detection="credential";{}'.format(filepath) + p.populate_set_out(msg, 'Tags') + #Put in form, count occurences, then send to moduleStats creds_sites = {} site_occurence = re.findall(regex_site_for_stats, content) diff --git a/bin/CreditCards.py b/bin/CreditCards.py index a7441807..260d1345 100755 --- a/bin/CreditCards.py +++ b/bin/CreditCards.py @@ -85,6 +85,9 @@ if __name__ == "__main__": #send to Browse_warning_paste msg = 'creditcard;{}'.format(filename) p.populate_set_out(msg, 'alertHandler') + + msg = 'infoleak:automatic-detection="credit-card";{}'.format(filename) + p.populate_set_out(msg, 'Tags') else: publisher.info('{}CreditCard related;{}'.format(to_print, paste.p_path)) else: diff --git a/bin/Cve.py b/bin/Cve.py index 9ac4efc8..bd240260 100755 --- a/bin/Cve.py +++ b/bin/Cve.py @@ -34,6 +34,9 @@ def search_cve(message): #send to Browse_warning_paste msg = 'cve;{}'.format(filepath) p.populate_set_out(msg, 'alertHandler') + + msg = 'infoleak:automatic-detection="cve";{}'.format(filepath) + p.populate_set_out(msg, 'Tags') #Send to duplicate p.populate_set_out(filepath, 'Duplicate') diff --git a/bin/Keys.py b/bin/Keys.py index 9f39cf50..7b1ec7dc 100755 --- a/bin/Keys.py +++ b/bin/Keys.py @@ -28,47 +28,76 @@ def search_key(paste): if '-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name)) + msg = 'infoleak:automatic-detection="pgp-message";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN CERTIFICATE-----' in content: publisher.warning('{} has a certificate message'.format(paste.p_name)) + + msg = 'infoleak:automatic-detection="certificate";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN RSA PRIVATE KEY-----' in content: publisher.warning('{} has a RSA private key message'.format(paste.p_name)) print('rsa private key message found') + + msg = 'infoleak:automatic-detection="rsa-private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN PRIVATE KEY-----' in content: publisher.warning('{} has a private key message'.format(paste.p_name)) print('private key message found') + + msg = 'infoleak:automatic-detection="private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content: publisher.warning('{} has an encrypted private key message'.format(paste.p_name)) print('encrypted private key message found') + + msg = 'infoleak:automatic-detection="encrypted-private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN OPENSSH PRIVATE KEY-----' in content: publisher.warning('{} has an openssh private key message'.format(paste.p_name)) print('openssh private key message found') + + msg = 'infoleak:automatic-detection="private-ssh-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN OpenVPN Static key V1-----' in content: publisher.warning('{} has an openssh private key message'.format(paste.p_name)) print('OpenVPN Static key message found') + + msg = 'infoleak:automatic-detection="vpn-static-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN DSA PRIVATE KEY-----' in content: publisher.warning('{} has a dsa private key message'.format(paste.p_name)) + + msg = 'infoleak:automatic-detection="dsa-private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN EC PRIVATE KEY-----' in content: publisher.warning('{} has an ec private key message'.format(paste.p_name)) + + msg = 'infoleak:automatic-detection="ec-private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN PGP PRIVATE KEY BLOCK-----' in content: publisher.warning('{} has a pgp private key block message'.format(paste.p_name)) + + msg = 'infoleak:automatic-detection="pgp-private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if find : diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index aca72e8e..9c372b37 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -110,6 +110,8 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' sleep 0.1 + #screen -S "Script_AIL" -X screen -t "Attributes" bash -c './Attributes.py; read x' + #sleep 0.1 screen -S "Script_AIL" -X screen -t "Lines" bash -c './Lines.py; read x' sleep 0.1 screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x' @@ -144,6 +146,8 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "Base64" bash -c './Base64.py; read x' sleep 0.1 + screen -S "Script_AIL" -X screen -t "DbDump" bash -c './DbDump.py; read x' + sleep 0.1 screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c './Bitcoin.py; read x' sleep 0.1 screen -S "Script_AIL" -X screen -t "Phone" bash -c './Phone.py; read x' @@ -160,6 +164,8 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "alertHandler" bash -c './alertHandler.py; read x' sleep 0.1 + screen -S "Script_AIL" -X screen -t "Tags" bash -c './Tags.py; read x' + sleep 0.1 screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' } diff --git a/bin/Mail.py b/bin/Mail.py index abc112a6..c1d8cf70 100755 --- a/bin/Mail.py +++ b/bin/Mail.py @@ -76,6 +76,9 @@ if __name__ == "__main__": p.populate_set_out(filename, 'Duplicate') p.populate_set_out('mail;{}'.format(filename), 'alertHandler') + msg = 'infoleak:automatic-detection="mail";{}'.format(filename) + p.populate_set_out(msg, 'Tags') + else: publisher.info(to_print) #Send to ModuleStats diff --git a/bin/Onion.py b/bin/Onion.py index 77ed75fe..277f1c71 100755 --- a/bin/Onion.py +++ b/bin/Onion.py @@ -152,6 +152,9 @@ if __name__ == "__main__": for url in fetch(p, r_cache, urls, domains_list, path): publisher.info('{}Checked {};{}'.format(to_print, url, PST.p_path)) p.populate_set_out('onion;{}'.format(PST.p_path), 'alertHandler') + + msg = 'infoleak:automatic-detection="onion";{}'.format(PST.p_path) + p.populate_set_out(msg, 'Tags') else: publisher.info('{}Onion related;{}'.format(to_print, PST.p_path)) diff --git a/bin/Phone.py b/bin/Phone.py index e3f0f908..213db2b3 100755 --- a/bin/Phone.py +++ b/bin/Phone.py @@ -36,6 +36,10 @@ def search_phone(message): msg = 'phone;{}'.format(message) p.populate_set_out(msg, 'alertHandler') #Send to duplicate + + msg = 'infoleak:automatic-detection="phone-number";{}'.format(message) + p.populate_set_out(msg, 'Tags') + p.populate_set_out(message, 'Duplicate') stats = {} for phone_number in results: diff --git a/bin/SQLInjectionDetection.py b/bin/SQLInjectionDetection.py index 9e28de72..117f3dc0 100755 --- a/bin/SQLInjectionDetection.py +++ b/bin/SQLInjectionDetection.py @@ -82,6 +82,9 @@ def analyse(url, path): p.populate_set_out(path, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler') + + msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path) + p.populate_set_out(msg, 'Tags') else: print("Potential SQL injection:") print(urllib.request.unquote(url)) diff --git a/bin/Tags.py b/bin/Tags.py new file mode 100755 index 00000000..f4939ec3 --- /dev/null +++ b/bin/Tags.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +""" +The Tags Module +================================ + +This module create tags. + +""" +import redis + +import time + +from pubsublogger import publisher +from Helper import Process +from packages import Paste + +if __name__ == '__main__': + + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + # Section name in bin/packages/modules.cfg + config_section = 'Tags' + + # Setup the I/O queues + p = Process(config_section) + + server = redis.StrictRedis( + host=p.config.get("ARDB_Tags", "host"), + port=p.config.get("ARDB_Tags", "port"), + db=p.config.get("ARDB_Tags", "db"), + decode_responses=True) + + server_metadata = redis.StrictRedis( + host=p.config.get("ARDB_Metadata", "host"), + port=p.config.get("ARDB_Metadata", "port"), + db=p.config.get("ARDB_Metadata", "db"), + decode_responses=True) + + # Sent to the logging a description of the module + publisher.info("Tags module started") + + # Endless loop getting messages from the input queue + while True: + # Get one message from the input queue + message = p.get_from_set() + + if message is None: + publisher.debug("{} queue is empty, waiting 10s".format(config_section)) + time.sleep(10) + continue + + else: + tag, path = message.split(';') + # add the tag to the tags word_list + res = server.sadd('list_tags', tag) + if res == 1: + print("new tags added : {}".format(tag)) + # add the path to the tag set + res = server.sadd(tag, path) + if res == 1: + print("new paste: {}".format(path)) + print(" tagged: {}".format(tag)) + server_metadata.sadd('tag:'+path, tag) diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py new file mode 100644 index 00000000..db501f67 --- /dev/null +++ b/var/www/modules/Tags/Flask_Tags.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the trending modules page +''' +import redis +from flask import Flask, render_template, jsonify, request, Blueprint + +import json + +import Paste + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg +r_serv_tags = Flask_config.r_serv_tags +r_serv_metadata = Flask_config.r_serv_metadata +max_preview_char = Flask_config.max_preview_char +max_preview_modal = Flask_config.max_preview_modal + +Tags = Blueprint('Tags', __name__, template_folder='templates') + +# ============ FUNCTIONS ============ +def one(): + return 1 + +# ============= ROUTES ============== + +@Tags.route("/Tags/", methods=['GET']) +def Tags_page(): + return render_template("Tags.html") + +@Tags.route("/Tags/get_all_tags") +def get_all_tags(): + + all_tags = r_serv_tags.smembers('list_tags') + + list_tags = [] + id = 0 + for tag in all_tags: + list_tags.append( tag ) + id += 1 + + return jsonify(list_tags) + +@Tags.route("/Tags/get_tagged_paste") +def get_tagged_paste(): + + tags = request.args.get('ltags')[1:-1] + tags = tags.replace('\\','') + + list_tags = tags.split(',') + tmp_list_tags = [] + + # remove " char + for tag in list_tags: + tmp_list_tags.append(tag[1:-1]) + list_tags = tmp_list_tags + + # TODO verify input + + if(type(list_tags) is list): + # no tag + if list_tags is False: + print('empty') + # 1 tag + elif len(list_tags) < 2: + tagged_pastes = r_serv_tags.smembers(list_tags[0]) + + # 2 tags or more + else: + tagged_pastes = r_serv_tags.sinter(list_tags[0], *list_tags[1:]) + + else : + return 'INCORRECT INPUT' + + #currentSelectYear = int(datetime.now().year) + currentSelectYear = 2018 + + bootstrap_label = [] + bootstrap_label.append('primary') + bootstrap_label.append('success') + bootstrap_label.append('danger') + bootstrap_label.append('warning') + bootstrap_label.append('info') + bootstrap_label.append('dark') + + all_content = [] + paste_date = [] + paste_linenum = [] + all_path = [] + allPastes = list(tagged_pastes) + paste_tags = [] + + for path in allPastes[0:50]: ######################moduleName + all_path.append(path) + paste = Paste.Paste(path) + content = paste.get_p_content() + content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 + all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " ")) + curr_date = str(paste._get_p_date()) + curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] + paste_date.append(curr_date) + paste_linenum.append(paste.get_lines_info()[0]) + p_tags = r_serv_metadata.smembers('tag:'+path) + l_tags = [] + for tag in p_tags: + tag = tag.split('=') + if len(tag) > 1: + if tag[1] != '': + tag = tag[1][1:-1] + # no value + else: + tag = tag[0][1:-1] + # use for custom tags + else: + tag = tag[0] + + l_tags.append(tag) + + paste_tags.append(l_tags) + + if len(allPastes) > 10: + finished = False + else: + finished = True + + return render_template("tagged.html", + year=currentSelectYear, + all_path=all_path, + paste_tags=paste_tags, + bootstrap_label=bootstrap_label, + content=all_content, + paste_date=paste_date, + paste_linenum=paste_linenum, + char_to_display=max_preview_modal, + finished=finished) + + return 'OK' + +@Tags.route("/Tags/res") +def get_tagged_paste_res(): + + return render_template("res.html") + +# ========= REGISTRATION ========= +app.register_blueprint(Tags) diff --git a/var/www/modules/Tags/templates/Tags.html b/var/www/modules/Tags/templates/Tags.html new file mode 100644 index 00000000..cb476749 --- /dev/null +++ b/var/www/modules/Tags/templates/Tags.html @@ -0,0 +1,78 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+
+
+

Tags

+
+ +
+ +
+
+ + +
+ +
+ +
+
+ + + +
+ + + + + + + diff --git a/var/www/modules/Tags/templates/header_Tags.html b/var/www/modules/Tags/templates/header_Tags.html new file mode 100644 index 00000000..624adb2f --- /dev/null +++ b/var/www/modules/Tags/templates/header_Tags.html @@ -0,0 +1 @@ +
  • Tags
    • diff --git a/var/www/modules/Tags/templates/tagged.html b/var/www/modules/Tags/templates/tagged.html new file mode 100644 index 00000000..17e5181b --- /dev/null +++ b/var/www/modules/Tags/templates/tagged.html @@ -0,0 +1,285 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + + + + +
    +
    +
    +

    Tags

    +
    + +
    + +
    +
    + + +
    + +
    + +
    +
    + + + + + + + + + + + + + + + + {% for path in all_path %} + + + + + + + + {% endfor %} + + +
    #PathDate# of linesAction
    {{ loop.index0 }}{{ path }} +
    + {% for tag in paste_tags[loop.index0] %} + {{ tag }} + {% endfor %} +
    +     		{{ paste_date[loop.index0] }}{{ paste_linenum[loop.index0] }}

    + +
    +
    + +
    +
    + + +
    +
    + +
    + + + + + + + diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py index 3a3be9be..398a350a 100644 --- a/var/www/modules/showpaste/Flask_showpaste.py +++ b/var/www/modules/showpaste/Flask_showpaste.py @@ -96,7 +96,17 @@ def showpaste(content_range): if content_range != 0: p_content = p_content[0:content_range] - return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list) + bootstrap_label = [] + bootstrap_label.append('primary') + bootstrap_label.append('success') + bootstrap_label.append('danger') + bootstrap_label.append('warning') + bootstrap_label.append('info') + bootstrap_label.append('dark') + + list_tags = r_serv_metadata.smembers('tag:'+requested_path) + + return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list) # ============ ROUTES ============ diff --git a/var/www/modules/showpaste/templates/show_saved_paste.html b/var/www/modules/showpaste/templates/show_saved_paste.html index b972ef12..2dae1caa 100644 --- a/var/www/modules/showpaste/templates/show_saved_paste.html +++ b/var/www/modules/showpaste/templates/show_saved_paste.html @@ -21,7 +21,13 @@

    Paste: {{ request.args.get('paste') }}

    -

    ({{ request.args.get('num') }})

    +

    +
    + {% for tag in list_tags %} + {{ tag }} + {% endfor %} +
    +