Merge branch 'master' of github.com:MISP/misp-modules into chrisr3d_patch

README.md

@@ -18,6 +18,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 ### Expansion modules
 
 * [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
+* [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP.
 * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
 * [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
 * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.

REQUIREMENTS

@@ -21,8 +21,10 @@ domaintools_api
 pygeoip
 bs4
 oauth2
-yara
+yara-python
 sigmatools
 stix2-patterns
 maclookup
-vulners
+vulners
+psutil
+blockchain

doc/documentation.md

@@ -10,6 +10,12 @@ Query an ASN description history service (https://github.com/CIRCL/ASN-Descripti
 
 -----
 
+#### [btc](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc.py)
+
+An expansion hover module to get a blockchain balance from a BTC address in MISP.
+
+-----
+
 #### [circl_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py)
 
 <img src=logos/passivedns.png height=60>

doc/expansion/btc.json

@@ -0,0 +1,3 @@
+{
+    "description": "An expansion hover module to get a blockchain balance from a BTC address in MISP."
+}

misp_modules/__init__.py

@@ -29,6 +29,7 @@ import fnmatch
 import argparse
 import re
 import datetime
+import psutil
 
 import tornado.web
 import tornado.process
@@ -241,7 +242,23 @@ def main():
     service = [(r'/modules', ListModules), (r'/query', QueryModule)]
 
     application = tornado.web.Application(service)
-    application.listen(port, address=listen)
+    try:
+        application.listen(port, address=listen)
+    except Exception as e:
+        if e.errno == 98:
+            pids = psutil.pids()
+            for pid in pids:
+                p = psutil.Process(pid)
+                if p.name() == "misp-modules":
+                    print("\n\n\n")
+                    print(e)
+                    print("\nmisp-modules is still running as PID: {}\n".format(pid))
+                    print("Please kill accordingly:")
+                    print("sudo kill {}".format(pid))
+                    sys.exit(-1)
+            print(e)
+            print("misp-modules might still be running.")
+
     log.info('MISP modules server started on {0} port {1}'.format(listen, port))
     if args.t:
         log.info('MISP modules started in test-mode, quitting immediately.')

misp_modules/modules/expansion/__init__.py

@@ -1,3 +1,3 @@
 from . import _vmray
 
-__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query']
+__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query']

misp_modules/modules/expansion/btc_steroids.py

@@ -0,0 +1,186 @@
+import sys
+import json
+import requests
+import time
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['btc'], 'output': ['text']}
+moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
+              'description': 'BTC expansion service to \
+                              get quick information from MISP attributes',
+              'module-type': ['hover']}
+
+moduleconfig = []
+
+blockchain_firstseen='https://blockchain.info/q/addressfirstseen/'
+blockchain_balance='https://blockchain.info/q/addressbalance/'
+blockchain_totalreceived='https://blockchain.info/q/getreceivedbyaddress/'
+blockchain_all='https://blockchain.info/rawaddr/'
+converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts='
+converter_rls = 'https://min-api.cryptocompare.com/stats/rate/limit'
+result_text = ""
+g_rate_limit = 300
+start_time = 0
+conversion_rates = {}
+
+def get_consumption(output=False):
+    req = requests.get(converter_rls)
+    jreq = req.json()
+    minute = str(jreq['Minute']['CallsLeft']['Histo'])
+    hour   = str(jreq['Hour']['CallsLeft']['Histo'])
+    # Debug out for the console
+    print("Calls left this minute / hour: " + minute + " / " + hour)
+    return minute, hour
+
+
+def convert(btc, timestamp):
+    global g_rate_limit
+    global start_time
+    global now
+    global conversion_rates
+    date = time.strftime('%Y-%m-%d', time.localtime(timestamp))
+    # Lookup conversion rates in the cache:
+    if date in conversion_rates:
+        (usd, eur) = conversion_rates[date]
+    else:
+        # If not cached, we have to get the converion rates
+        # We have to be careful with rate limiting on the server side
+        if g_rate_limit == 300:
+            minute, hour = get_consumption()
+        g_rate_limit -= 1
+        now = time.time()
+        delta = now - start_time
+        #print(g_rate_limit)
+        if g_rate_limit <= 10:
+            minute, hour = get_consumption(output=True)
+            if int(minute) <= 10:
+                #print(minute)
+                #get_consumption(output=True)
+                time.sleep(3)
+            else:
+                mprint(minute)
+                start_time = time.time()
+                g_rate_limit = int(minute)
+        try:
+            req = requests.get(converter+str(timestamp))
+            jreq = req.json()
+            usd = jreq['BTC']['USD']
+            eur = jreq['BTC']['EUR']
+            # Since we have the rates, store them in the cache
+            conversion_rates[date] = (usd, eur)
+        except Exception as ex:
+            mprint(ex)
+            get_consumption(output=True)
+    # Actually convert and return the values
+    u = usd * btc
+    e = eur * btc
+    return u,e
+
+
+def mprint(input):
+    # Prepare the final print
+    global result_text
+    result_text = result_text + "\n" + str(input)
+
+
+def handler(q=False):
+    global result_text
+    global conversion_rates
+    start_time = time.time()
+    now = time.time()
+    if q is False:
+        return False
+    request = json.loads(q)
+    click = False
+    # This means the magnifying glass has been clicked
+    if request.get('persistent') == 1:
+        click = True
+    # Otherwise the attribute was only hovered over
+    if request.get('btc'):
+        btc = request['btc']
+    else:
+        return False
+
+    mprint("\nAddress:\t" + btc)
+    try:
+        req = requests.get(blockchain_all+btc+"?limit=50&filter=5")
+        jreq = req.json()
+    except Exception as e:
+        #print(e)
+        print(req.text)
+        result_text = ""
+        sys.exit(1) 
+
+    n_tx = jreq['n_tx']
+    balance = float(jreq['final_balance'] / 100000000)
+    rcvd = float(jreq['total_received'] / 100000000)
+    sent = float(jreq['total_sent'] / 100000000)
+    output = 'Balance:\t{0:.10f} BTC (+{1:.10f} BTC / -{2:.10f} BTC)'
+    mprint(output.format(balance, rcvd, sent))
+    if click is False:
+        mprint("Transactions:\t" + str(n_tx) + "\t (previewing up to 5 most recent)")
+    else:
+        mprint("Transactions:\t" + str(n_tx))
+    mprint("======================================================================================")
+    i = 0
+    while i < n_tx:
+        if click is False:
+            req = requests.get(blockchain_all+btc+"?limit=5&offset="+str(i)+"&filter=5")
+            if n_tx > 5:
+                n_tx = 5
+        else:
+            req = requests.get(blockchain_all+btc+"?limit=50&offset="+str(i)+"&filter=5")
+        jreq = req.json()
+        if jreq['txs']:
+            for transactions in jreq['txs']:
+                sum = 0
+                sum_counter = 0
+                for tx in transactions['inputs']:
+                    script_old = tx['script']
+                    if tx['prev_out']['value'] != 0 and tx['prev_out']['addr'] == btc:
+                        datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time'])))
+                        value = float(tx['prev_out']['value'] / 100000000 )
+                        u,e = convert(value, transactions['time'])
+                        mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0'))
+                        if script_old != tx['script']:
+                            i += 1
+                        else:
+                            sum_counter += 1
+                            sum += value
+                if sum_counter > 1:
+                    u,e = convert(sum, transactions['time'])
+                    mprint("\t\t\t\t\t----------------------------------------------")
+                    mprint("#" + str(n_tx - i) + "\t\t\t\t  Sum:\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR\n".format(sum, u, e).rstrip('0'))
+                for tx in transactions['out']:
+                    if tx['value'] != 0 and tx['addr'] == btc:
+                        datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time'])))
+                        value = float(tx['value'] / 100000000 )
+                        u,e = convert(value, transactions['time'])
+                        mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t {0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0'))
+                        #i += 1
+                i += 1
+
+    r = { 
+        'results': [
+            {
+                'types': ['text'],
+                'values':[
+                    str(result_text)
+                ]
+            }
+        ]
+    }
+    # Debug output on the console
+    print(result_text)
+    # Unset the result for the next request
+    result_text = ""
+    return r
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/onyphe.py

@@ -65,16 +65,16 @@ def handle_expansion(api, ip, misperrors):
 
     for r in result['results']:
         if r['@category'] == 'pastries':
-            if r['@type'] == 'pastebin':
+            if r['source'] == 'pastebin':
                 urls_pasties.append('https://pastebin.com/raw/%s' % r['key'])
         elif r['@category'] == 'synscan':
             asn_list.append(r['asn'])
             os_target = r['os']
             if os_target != 'Unknown':
                 os_list.append(r['os'])
-        elif r['@category'] == 'resolver' and r['@type'] =='reverse':
+        elif r['@category'] == 'resolver' and r['type'] =='reverse':
             domains_resolver.append(r['reverse'])
-        elif r['@category'] == 'resolver' and r['@type'] =='forward':
+        elif r['@category'] == 'resolver' and r['type'] =='forward':
             domains_forward.append(r['forward'])
 
     result_filtered['results'].append({'types': ['url'], 'values': urls_pasties,
@@ -105,4 +105,4 @@ def introspection():
 
 def version():
     moduleinfo['config'] = moduleconfig
-    return moduleinfo
+    return moduleinfo

misp_modules/modules/expansion/onyphe_full.py

@@ -315,7 +315,7 @@ def expand_pastries(api, misperror, **kwargs):
         status_ok = True
         for item in result['results']:
             if item['@category'] == 'pastries':
-                if item['@type'] == 'pastebin':
+                if item['source'] == 'pastebin':
                     urls_pasties.append('https://pastebin.com/raw/%s' % item['key'])
 
                     if 'domain' in item:
@@ -374,4 +374,4 @@ def introspection():
 
 def version():
     moduleinfo['config'] = moduleconfig
-    return moduleinfo
+    return moduleinfo

tests/bodyhashdd.json

@@ -1 +1 @@
-{"module": "hashdd", "md5": "838DE99E82C5B9753BAC96D82C1A8DCB"}
+{"module": "hashdd", "md5": "838DE99E82C5B9753BAC96D82C1A8DCC"}