add bitcoin module

bin/Base64.py

@@ -18,7 +18,8 @@ import base64
 from hashlib import sha1
 import magic
 
-def search_base64(content):
+def search_base64(content, message):
+    find = False
     base64_list = re.findall(regex_base64, content)
     if(len(base64_list) > 0):
 
@@ -30,7 +31,16 @@ def search_base64(content):
                 #print(type)
                 #print(decode)
 
+                find = True
                 save_base64_as_file(decode, type)
+                print('found {} '.format(type))
+    if(find):
+        publisher.warning('base64 decoded')
+        #Send to duplicate
+        p.populate_set_out(message, 'Duplicate')
+        #send to Browse_warning_paste
+        msg = ('base64;{}'.format(message))
+        p.populate_set_out( msg, 'alertHandler')
 
 def save_base64_as_file(decode, type):
 
@@ -85,7 +95,7 @@ if __name__ == '__main__':
         content = paste.get_p_content()
 
         #print(filename)
-        search_base64(content)
+        search_base64(content,message)
 
         # (Optional) Send that thing to the next queue
         #p.populate_set_out(something_has_been_done)

bin/Bitcoin.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3.5
+# -*-coding:UTF-8 -*
+"""
+The Bitcoin Module
+============================
+
+It trying to extract Bitcoin address and secret key from paste
+
+    ..seealso:: Paste method (get_regex)
+
+Requirements
+------------
+
+*Need running Redis instances. (Redis).
+
+"""
+
+from packages import Paste
+from Helper import Process
+from pubsublogger import publisher
+
+import re
+import time
+
+from hashlib import sha256
+
+
+# thank http://rosettacode.org/wiki/Bitcoin/address_validation#Python for this 2 functions
+
+def decode_base58(bc, length):
+    n = 0
+    for char in bc:
+        n = n * 58 + digits58.index(char)
+    return n.to_bytes(length, 'big')
+def check_bc(bc):
+    try:
+        bcbytes = decode_base58(bc, 25)
+        return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
+    except Exception:
+        return False
+########################################################3
+
+def search_key(content, message):
+    bitcoin_address = re.findall(regex_bitcoin_public_address, content)
+    bitcoin_private_key = re.findall(regex_bitcoin_private_key, content)
+    validate_address = False
+    if(len(bitcoin_address) >0):
+        #print(message)
+        for address in bitcoin_address:
+            if(check_bc(address)):
+                validate_address = True
+                print('Bitcoin address found : {}'.format(address))
+                if(len(bitcoin_private_key) > 0):
+                    for private_key in bitcoin_private_key:
+                        print('Bitcoin private key found : {}'.format(private_key))
+
+        if(validate_address):
+            p.populate_set_out(message, 'Duplicate')
+            to_print = 'bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key))
+            print(to_print)
+            publisher.warning(to_print)
+            msg = ('bitcoin;{}'.format(message))
+            p.populate_set_out( msg, 'alertHandler')
+
+if __name__ == "__main__":
+    publisher.port = 6380
+    publisher.channel = "Script"
+
+    config_section = 'Bitcoin'
+
+    # Setup the I/O queues
+    p = Process(config_section)
+
+    # Sent to the logging a description of the module
+    publisher.info("Run Keys module ")
+
+    digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+
+    regex_bitcoin_public_address = re.compile(r'(?<![a-km-zA-HJ-NP-Z0-9])[13][a-km-zA-HJ-NP-Z0-9]{26,33}(?![a-km-zA-HJ-NP-Z0-9])')
+    regex_bitcoin_private_key = re.compile(r'[5KL][1-9A-HJ-NP-Za-km-z]{50,51}')
+
+    # Endless loop getting messages from the input queue
+    while True:
+        # Get one message from the input queue
+        message = p.get_from_set()
+        if message is None:
+            publisher.debug("{} queue is empty, waiting".format(config_section))
+            time.sleep(1)
+            continue
+
+        # Do something with the message from the queue
+        paste = Paste.Paste(message)
+        content = paste.get_p_content()
+        search_key(content, message)

bin/packages/config.cfg.backup

@@ -0,0 +1,148 @@
+[Directories]
+bloomfilters = Blooms
+dicofilters = Dicos
+pastes = PASTES
+base64 = BASE64
+wordtrending_csv = var/www/static/csv/wordstrendingdata
+wordsfile = files/wordfile
+protocolstrending_csv = var/www/static/csv/protocolstrendingdata
+protocolsfile = files/protocolsfile
+tldstrending_csv = var/www/static/csv/tldstrendingdata
+tldsfile = faup/src/data/mozilla.tlds
+domainstrending_csv = var/www/static/csv/domainstrendingdata
+pystemonpath = /home/pystemon/pystemon/
+sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
+
+[Notifications]
+sender = sender@example.com
+sender_host = smtp.example.com
+sender_port = 1337
+
+[Flask]
+max_preview_char = 250
+max_preview_modal = 800
+default_display = 10
+minute_processed_paste = 10
+diffmaxlinelength = 10000
+
+[Categ]
+matchingthreshold = 1
+
+[Credential]
+minimumlengththreshold = 3
+criticalnumbertoalert = 8
+mintoppasslist = 5
+
+[Modules_Duplicates]
+maximum_month_range = 3
+threshold_duplicate_ssdeep = 50
+threshold_duplicate_tlsh = 100
+min_paste_size = 0.3
+
+[Module_ModuleInformation]
+threshold_stucked_module = 600
+
+[Module_Mixer]
+operation_mode = 3
+ttl_duplicate = 86400
+
+[Module_Base64]
+path = Base64/
+
+[Redis_Cache]
+host = localhost
+port = 6379
+db = 0
+
+[Redis_Log]
+host = localhost
+port = 6380
+db = 0
+
+[Redis_Queues]
+host = localhost
+port = 6381
+db = 0
+
+[Redis_Data_Merging]
+host = localhost
+port = 6379
+db = 1
+
+[Redis_Paste_Name]
+host = localhost
+port = 6379
+db = 2
+
+[Redis_Mixer_Cache]
+host = localhost
+port = 6381
+db = 1
+
+[Redis_Level_DB_Curve]
+host = localhost
+port = 6382
+db = 1
+
+[Redis_Level_DB_Sentiment]
+host = localhost
+port = 6382
+db = 4
+
+[Redis_Level_DB_TermFreq]
+host = localhost
+port = 6382
+db = 2
+
+[Redis_Level_DB_TermCred]
+host = localhost
+port = 6382
+db = 5
+
+[Redis_Level_DB]
+host = localhost
+db = 0
+
+[Redis_Level_DB_Trending]
+host = localhost
+port = 6382
+db = 3
+
+[Redis_Level_DB_Hashs]
+host = localhost
+db = 1
+
+[Url]
+cc_critical = DE
+
+[DomClassifier]
+cc = DE
+cc_tld = r'\.de$'
+
+[Indexer]
+type = whoosh
+path = indexdir
+register = indexdir/all_index.txt
+index_max_size = 2000
+
+[ailleakObject]
+maxduplicatetopushtomisp = 10
+
+[ZMQ_Global]
+address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
+channel = 102
+bind = tcp://127.0.0.1:5556
+
+[ZMQ_Url]
+address = tcp://127.0.0.1:5004
+channel = urls
+
+[ZMQ_FetchedOnion]
+address = tcp://127.0.0.1:5005
+channel = FetchedOnion
+
+[RedisPubSub]
+host = localhost
+port = 6381
+db = 0
+

bin/packages/modules.cfg

@@ -112,4 +112,8 @@ publish = Redis_Duplicate,Redis_alertHandler
 
 [Base64]
 subscribe = Redis_Global
-#publish = ZMQ_Global
+publish = Redis_Duplicate,Redis_alertHandler
+
+[Bitcoin]
+subscribe = Redis_Global
+publish = Redis_Duplicate,Redis_alertHandler

var/www/modules/browsepastes/templates/browse_important_paste.html

@@ -90,6 +90,8 @@
         <li name='nav-pan'><a data-toggle="tab" href="#mail-tab" data-attribute-name="mail" data-panel="mail-panel">Mails</a></li>
         <li name='nav-pan'><a data-toggle="tab" href="#phone-tab" data-attribute-name="phone" data-panel="phone-panel">Phones</a></li>
         <li name='nav-pan'><a data-toggle="tab" href="#onion-tab" data-attribute-name="onion" data-panel="onion-panel">Onions</a></li>
+        <li name='nav-pan'><a data-toggle="tab" href="#bitcoin-tab" data-attribute-name="bitcoin" data-panel="bitcoin-panel">Bitcoin</a></li>
+        <li name='nav-pan'><a data-toggle="tab" href="#base64-tab" data-attribute-name="base64" data-panel="base64-panel">Base64</a></li>
     </ul>
     </br>
 
@@ -119,6 +121,12 @@
         <div class="col-lg-12 tab-pane fade" id="onion-tab">
             <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
         </div>
+        <div class="col-lg-12 tab-pane fade" id="bitcoin-tab">
+            <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
+        </div>
+        <div class="col-lg-12 tab-pane fade" id="base64-tab">
+            <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
+        </div>
     </div> <!-- tab-content -->
     <!-- /.row -->
 </div>

var/www/modules/browsepastes/templates/important_paste_by_module.html

@@ -245,4 +245,3 @@ $(document).ready(function(){
    } );
 
 </script>
-