chg: [Bitcoin] map cryptocurrency: bitcoin (DB pivot)

OVERVIEW.md

@@ -202,6 +202,38 @@ Redis and ARDB overview
 | | |
 | item_pgp_mail:*item_path* | *mail* |
 
+#### Cryptocurrency
+
+Supported cryptocurrency:
+- bitcoin
+
+##### Hset:
+| Key | Field | Value |
+| ------ | ------ | ------ |
+| cryptocurrency_metadata_**cryptocurrency name**:**cryptocurrency address** | first_seen | **date** |
+| | last_seen | **date** |
+
+##### set:
+| Key | Value |
+| ------ | ------ |
+| set_cryptocurrency_**cryptocurrency name**:**cryptocurrency address** | **item_path** |
+
+##### Hset date:
+| Key | Field | Value |
+| ------ | ------ |
+| cryptocurrency_**cryptocurrency name**:**date** | **cryptocurrency address** | **nb seen** |
+
+##### zset:
+| Key | Field | Value |
+| ------ | ------ | ------ |
+| cryptocurrency_all:**cryptocurrency name** | **cryptocurrency address** | **nb seen** |
+
+##### set:
+| Key | Value |
+| ------ | ------ |
+| item_cryptocurrency_**cryptocurrency name**:**item_path** | **cryptocurrency address** |
+
+
 ## DB9 - Crawler:
 
 ##### Hset:

bin/Bitcoin.py

@@ -44,6 +44,7 @@ def check_bc(bc):
 def search_key(content, message, paste):
     bitcoin_address = re.findall(regex_bitcoin_public_address, content)
     bitcoin_private_key = re.findall(regex_bitcoin_private_key, content)
+    date = str(paste._get_p_date())
     validate_address = False
     key = False
     if(len(bitcoin_address) >0):
@@ -56,6 +57,8 @@ def search_key(content, message, paste):
                     for private_key in bitcoin_private_key:
                         print('Bitcoin private key found : {}'.format(private_key))
                         key = True
+                # build bitcoin correlation
+                save_bitcoin_data(date, message, address):
 
         if(validate_address):
             p.populate_set_out(message, 'Duplicate')
@@ -75,6 +78,31 @@ def search_key(content, message, paste):
                 publisher.warning('{}Detected {} Bitcoin private key;{}'.format(
                     to_print, len(bitcoin_private_key),paste.p_rel_path))
 
+def save_bitcoin_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
+    # create basic medata
+    if not serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
+        serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date)
+        serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
+    else:
+        last_seen = serv_metadata.hget('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen')
+        if not last_seen:
+            serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
+        else:
+            if int(last_seen) < int(date):
+                serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
+
+    # global set
+    serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path)
+
+    # daily
+    serv_metadata.hincrby('cryptocurrency_{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1)
+
+    # all type
+    serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1)
+
+    # item_metadata
+    serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address)
+
 if __name__ == "__main__":
     publisher.port = 6380
     publisher.channel = "Script"

var/www/modules/hashDecoded/Flask_hashDecoded.py

@@ -13,6 +13,8 @@ from Date import Date
 from io import BytesIO
 import zipfile
 
+from hashlib import sha256
+
 import requests
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, send_file
 
@@ -158,6 +160,38 @@ def get_all_pgp_from_item(item_path):
 def one():
     return 1
 
+def decode_base58(bc, length):
+    n = 0
+    for char in bc:
+        n = n * 58 + digits58.index(char)
+    return n.to_bytes(length, 'big')
+
+def check_bc(bc):
+    try:
+        bcbytes = decode_base58(bc, 25)
+        return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
+    except Exception:
+        return False
+
+def get_bitcoin_address_metadata(bitcoin_address):
+    address_metadata = {}
+    if r_serv_metadata.exists('bitcoin_metadata:{}'.format(bitcoin_address)):
+        address_metadata['first_seen'] = r_serv_metadata.hget('bitcoin_metadata:{}'.format(bitcoin_address), 'first_seen')
+        address_metadata['first_seen'] = '{}/{}/{}'.format(address_metadata['first_seen'][0:4], address_metadata['first_seen'][4:6], address_metadata['first_seen'][6:8])
+        address_metadata['last_seen'] = r_serv_metadata.hget('bitcoin_metadata:{}'.format(bitcoin_address), 'last_seen')
+        address_metadata['last_seen'] = '{}/{}/{}'.format(address_metadata['last_seen'][0:4], address_metadata['last_seen'][4:6], address_metadata['last_seen'][6:8])
+        address_metadata['nb_seen'] = r_serv_metadata.scard('bitcoin:{}'.format(bitcoin_address))
+    return address_metadata
+
+def list_sparkline_bitcoin_values(date_range_sparkline, bitcoin_address):
+    sparklines_value = []
+    for date_day in date_range_sparkline:
+        nb_seen_this_day = r_serv_metadata.hget('bitcoin:{}'.format(bitcoin_address), bitcoin_address)
+        if nb_seen_this_day is None:
+            nb_seen_this_day = 0
+        sparklines_value.append(int(nb_seen_this_day))
+    return sparklines_value
+
 # ============= ROUTES ==============
 @hashDecoded.route("/hashDecoded/all_hash_search", methods=['POST'])
 def all_hash_search():
@@ -777,7 +811,7 @@ def update_vt_result():
         # TODO FIXME make json response
         return jsonify()
 
-## PGPDump ##
+############################ PGPDump ############################
 
 @hashDecoded.route("/decoded/pgpdump", methods=['GET'])
 def pgpdump_page():
@@ -1069,5 +1103,29 @@ def pgp_by_type_json():
     else:
         return jsonify()
 
+############################ Bitcoin ############################
+'''
+@hashDecoded.route('/correlation/show_bitcoin_address')
+def show_bitcoin_address():
+    bitcoin_address = request.args.get('bitcoin_address')
+
+    # validate user input
+    if check_bc(bitcoin_address):
+        bitcoin_address_metadata = get_bitcoin_address_metadata(bitcoin_address)
+        if bitcoin_address_metadata:
+
+            num_day_sparkline = 6
+            date_range_sparkline = get_date_range(num_day_sparkline)
+
+            sparkline_values = list_sparkline_bitcoin_values(date_range_sparkline, bitcoin_address)
+            return render_template('showPgpDump.html', bitcoin_address=bitcoin_address,
+                            key_id_metadata=bitcoin_address_metadata,
+                            sparkline_values=sparkline_values)
+        else:
+            return '404'
+    else:
+        return 'error'
+'''
+
 # ========= REGISTRATION =========
 app.register_blueprint(hashDecoded, url_prefix=baseUrl)