diff --git a/README.md b/README.md index 753c15c7..f8cb8f62 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ Features * Multiple feed support * Each module can process and reprocess the information already processed by AIL * Detecting and extracting URLs including their geographical location (e.g. IP address location) -* Extracting and validating potential leak of credit cards numbers, credentials, ... -* Extracting and validating email addresses leaked including DNS MX validation +* Extracting and validating potential leaks of credit card numbers, credentials, ... +* Extracting and validating leaked email addresses, including DNS MX validation * Module for extracting Tor .onion addresses (to be further processed for analysis) * Keep tracks of duplicates (and diffing between each duplicate found) * Extracting and validating potential hostnames (e.g. to feed Passive DNS systems) diff --git a/bin/Bitcoin.py b/bin/Bitcoin.py deleted file mode 100755 index a3cfcfc7..00000000 --- a/bin/Bitcoin.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* -""" -The Bitcoin Module -============================ - -It trying to extract Bitcoin address and secret key from paste - - ..seealso:: Paste method (get_regex) - -Requirements ------------- - -*Need running Redis instances. (Redis). - -""" - -from packages import Paste -from Helper import Process -from pubsublogger import publisher - -import re -import time -import redis - -from hashlib import sha256 - - -#### thank http://rosettacode.org/wiki/Bitcoin/address_validation#Python for this 2 functions - -def decode_base58(bc, length): - n = 0 - for char in bc: - n = n * 58 + digits58.index(char) - return n.to_bytes(length, 'big') - -def check_bc(bc): - try: - bcbytes = decode_base58(bc, 25) - return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4] - except Exception: - return False -######################################################## - -def search_key(content, message, paste): - bitcoin_address = re.findall(regex_bitcoin_public_address, content) - bitcoin_private_key = re.findall(regex_bitcoin_private_key, content) - date = str(paste._get_p_date()) - validate_address = False - key = False - if(len(bitcoin_address) >0): - #print(message) - for address in bitcoin_address: - if(check_bc(address)): - validate_address = True - print('Bitcoin address found : {}'.format(address)) - if(len(bitcoin_private_key) > 0): - for private_key in bitcoin_private_key: - print('Bitcoin private key found : {}'.format(private_key)) - key = True - # build bitcoin correlation - save_cryptocurrency_data('bitcoin', date, message, address) - - if(validate_address): - p.populate_set_out(message, 'Duplicate') - to_print = 'Bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key)) - print(to_print) - publisher.warning(to_print) - - msg = 'infoleak:automatic-detection="bitcoin-address";{}'.format(message) - p.populate_set_out(msg, 'Tags') - - if(key): - msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message) - p.populate_set_out(msg, 'Tags') - - to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date, - paste.p_name) - publisher.warning('{}Detected {} Bitcoin private key;{}'.format( - to_print, len(bitcoin_private_key),paste.p_rel_path)) - -def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address): - # create basic medata - if not serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)): - serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date) - serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date) - else: - last_seen = serv_metadata.hget('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen') - if not last_seen: - serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date) - else: - if int(last_seen) < int(date): - serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date) - - # global set - serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path) - - # daily - serv_metadata.hincrby('cryptocurrency:{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1) - - # all type - serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1) - - # item_metadata - serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address) - -if __name__ == "__main__": - publisher.port = 6380 - publisher.channel = "Script" - - config_section = 'Bitcoin' - - # Setup the I/O queues - p = Process(config_section) - - serv_metadata = redis.StrictRedis( - host=p.config.get("ARDB_Metadata", "host"), - port=p.config.getint("ARDB_Metadata", "port"), - db=p.config.getint("ARDB_Metadata", "db"), - decode_responses=True) - - # Sent to the logging a description of the module - publisher.info("Run Keys module ") - - digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' - - regex_bitcoin_public_address = re.compile(r'(? 0: + print('{} contains {} IPs'.format(paste.p_name, len(matching_ips))) + publisher.warning('{} contains {} IPs'.format(paste.p_name, len(matching_ips))) + + #Tag message with IP + msg = 'infoleak:automatic-detection="ip";{}'.format(message) + p.populate_set_out(msg, 'Tags') + #Send to duplicate + p.populate_set_out(message, 'Duplicate') + +if __name__ == '__main__': + # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + # Section name in bin/packages/modules.cfg + config_section = 'IP' + # Setup the I/O queues + p = Process(config_section) + + ip_networks = [] + for network in p.config.get("IP", "networks").split(","): + ip_networks.append(IPv4Network(network)) + + + # Sent to the logging a description of the module + publisher.info("Run IP module") + + # Endless loop getting messages from the input queue + while True: + # Get one message from the input queue + message = p.get_from_set() + if message is None: + publisher.debug("{} queue is empty, waiting".format(config_section)) + time.sleep(1) + continue + + # Do something with the message from the queue + search_ip(message) + diff --git a/bin/Keys.py b/bin/Keys.py index eb06601a..237f807c 100755 --- a/bin/Keys.py +++ b/bin/Keys.py @@ -121,6 +121,13 @@ def search_key(paste): p.populate_set_out(msg, 'Tags') find = True + if '-----BEGIN PUBLIC KEY-----' in content: + publisher.warning('{} has a public key message'.format(paste.p_name)) + + msg = 'infoleak:automatic-detection="public-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') + find = True + # pgp content if get_pgp_content: p.populate_set_out(message, 'PgpDump') diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 71621a8e..406c4a77 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -186,7 +186,9 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Bitcoin.py; read x" + screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tools.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x" sleep 0.1 @@ -213,6 +215,8 @@ function launching_scripts { screen -S "Script_AIL" -X screen -t "UpdateBackground" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./update-background.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x" } diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 8e551697..2dc94e38 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -132,3 +132,7 @@ publish = Redis_Mixer [Crawler] subscribe = Redis_Crawler publish = Redis_Mixer,Redis_Tags + +[IP] +subscribe = Redis_Global +publish = Redis_Duplicate,Redis_Tags diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index c59e5c6b..031927eb 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -266,3 +266,8 @@ default_crawler_closespider_pagecount = 50 default_crawler_user_agent = Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0 splash_url = http://127.0.0.1 splash_port = 8050-8052 + +[IP] +# list of comma-separated CIDR that you wish to be alerted for. e.g: +#networks = 192.168.34.0/24,10.0.0.0/8,192.168.33.0/24 +networks = diff --git a/requirements.txt b/requirements.txt index dbebad0f..70eab9a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,6 +50,7 @@ flask-login bcrypt #DomainClassifier +git+https://github.com/D4-project/BGP-Ranking.git/#egg=pybgpranking&subdirectory=client DomainClassifier #Indexer requirements whoosh