From ff23a2bce2b7fb06ca17d0826f5b4217cd5b20f4 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 21 May 2019 16:14:09 +0200 Subject: [PATCH] chg: [Bitcoin] map cryptocurrency: bitcoin (DB pivot) --- OVERVIEW.md | 32 ++++++++++ bin/Bitcoin.py | 28 +++++++++ .../modules/hashDecoded/Flask_hashDecoded.py | 60 ++++++++++++++++++- 3 files changed, 119 insertions(+), 1 deletion(-) diff --git a/OVERVIEW.md b/OVERVIEW.md index 2b357f9f..ab413a0c 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -202,6 +202,38 @@ Redis and ARDB overview | | | | item_pgp_mail:*item_path* | *mail* | +#### Cryptocurrency + +Supported cryptocurrency: +- bitcoin + +##### Hset: +| Key | Field | Value | +| ------ | ------ | ------ | +| cryptocurrency_metadata_**cryptocurrency name**:**cryptocurrency address** | first_seen | **date** | +| | last_seen | **date** | + +##### set: +| Key | Value | +| ------ | ------ | +| set_cryptocurrency_**cryptocurrency name**:**cryptocurrency address** | **item_path** | + +##### Hset date: +| Key | Field | Value | +| ------ | ------ | +| cryptocurrency_**cryptocurrency name**:**date** | **cryptocurrency address** | **nb seen** | + +##### zset: +| Key | Field | Value | +| ------ | ------ | ------ | +| cryptocurrency_all:**cryptocurrency name** | **cryptocurrency address** | **nb seen** | + +##### set: +| Key | Value | +| ------ | ------ | +| item_cryptocurrency_**cryptocurrency name**:**item_path** | **cryptocurrency address** | + + ## DB9 - Crawler: ##### Hset: diff --git a/bin/Bitcoin.py b/bin/Bitcoin.py index ff76c5f0..a84aae3c 100755 --- a/bin/Bitcoin.py +++ b/bin/Bitcoin.py @@ -44,6 +44,7 @@ def check_bc(bc): def search_key(content, message, paste): bitcoin_address = re.findall(regex_bitcoin_public_address, content) bitcoin_private_key = re.findall(regex_bitcoin_private_key, content) + date = str(paste._get_p_date()) validate_address = False key = False if(len(bitcoin_address) >0): @@ -56,6 +57,8 @@ def search_key(content, message, paste): for private_key in bitcoin_private_key: print('Bitcoin private key found : {}'.format(private_key)) key = True + # build bitcoin correlation + save_bitcoin_data(date, message, address): if(validate_address): p.populate_set_out(message, 'Duplicate') @@ -75,6 +78,31 @@ def search_key(content, message, paste): publisher.warning('{}Detected {} Bitcoin private key;{}'.format( to_print, len(bitcoin_private_key),paste.p_rel_path)) +def save_bitcoin_data(cryptocurrency_name, date, item_path, cryptocurrency_address): + # create basic medata + if not serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)): + serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date) + serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date) + else: + last_seen = serv_metadata.hget('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen') + if not last_seen: + serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date) + else: + if int(last_seen) < int(date): + serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date) + + # global set + serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path) + + # daily + serv_metadata.hincrby('cryptocurrency_{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1) + + # all type + serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1) + + # item_metadata + serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address) + if __name__ == "__main__": publisher.port = 6380 publisher.channel = "Script" diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index 29a48726..28747bc2 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -13,6 +13,8 @@ from Date import Date from io import BytesIO import zipfile +from hashlib import sha256 + import requests from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, send_file @@ -158,6 +160,38 @@ def get_all_pgp_from_item(item_path): def one(): return 1 +def decode_base58(bc, length): + n = 0 + for char in bc: + n = n * 58 + digits58.index(char) + return n.to_bytes(length, 'big') + +def check_bc(bc): + try: + bcbytes = decode_base58(bc, 25) + return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4] + except Exception: + return False + +def get_bitcoin_address_metadata(bitcoin_address): + address_metadata = {} + if r_serv_metadata.exists('bitcoin_metadata:{}'.format(bitcoin_address)): + address_metadata['first_seen'] = r_serv_metadata.hget('bitcoin_metadata:{}'.format(bitcoin_address), 'first_seen') + address_metadata['first_seen'] = '{}/{}/{}'.format(address_metadata['first_seen'][0:4], address_metadata['first_seen'][4:6], address_metadata['first_seen'][6:8]) + address_metadata['last_seen'] = r_serv_metadata.hget('bitcoin_metadata:{}'.format(bitcoin_address), 'last_seen') + address_metadata['last_seen'] = '{}/{}/{}'.format(address_metadata['last_seen'][0:4], address_metadata['last_seen'][4:6], address_metadata['last_seen'][6:8]) + address_metadata['nb_seen'] = r_serv_metadata.scard('bitcoin:{}'.format(bitcoin_address)) + return address_metadata + +def list_sparkline_bitcoin_values(date_range_sparkline, bitcoin_address): + sparklines_value = [] + for date_day in date_range_sparkline: + nb_seen_this_day = r_serv_metadata.hget('bitcoin:{}'.format(bitcoin_address), bitcoin_address) + if nb_seen_this_day is None: + nb_seen_this_day = 0 + sparklines_value.append(int(nb_seen_this_day)) + return sparklines_value + # ============= ROUTES ============== @hashDecoded.route("/hashDecoded/all_hash_search", methods=['POST']) def all_hash_search(): @@ -777,7 +811,7 @@ def update_vt_result(): # TODO FIXME make json response return jsonify() -## PGPDump ## +############################ PGPDump ############################ @hashDecoded.route("/decoded/pgpdump", methods=['GET']) def pgpdump_page(): @@ -1069,5 +1103,29 @@ def pgp_by_type_json(): else: return jsonify() +############################ Bitcoin ############################ +''' +@hashDecoded.route('/correlation/show_bitcoin_address') +def show_bitcoin_address(): + bitcoin_address = request.args.get('bitcoin_address') + + # validate user input + if check_bc(bitcoin_address): + bitcoin_address_metadata = get_bitcoin_address_metadata(bitcoin_address) + if bitcoin_address_metadata: + + num_day_sparkline = 6 + date_range_sparkline = get_date_range(num_day_sparkline) + + sparkline_values = list_sparkline_bitcoin_values(date_range_sparkline, bitcoin_address) + return render_template('showPgpDump.html', bitcoin_address=bitcoin_address, + key_id_metadata=bitcoin_address_metadata, + sparkline_values=sparkline_values) + else: + return '404' + else: + return 'error' +''' + # ========= REGISTRATION ========= app.register_blueprint(hashDecoded, url_prefix=baseUrl)