diff --git a/README.md b/README.md index b8bd14d..01ec367 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ### Expansion modules * [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history. +* [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP. * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen. * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. diff --git a/REQUIREMENTS b/REQUIREMENTS index 6ab46cc..cfaf9ad 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -21,8 +21,10 @@ domaintools_api pygeoip bs4 oauth2 -yara +yara-python sigmatools stix2-patterns maclookup -vulners \ No newline at end of file +vulners +psutil +blockchain diff --git a/doc/documentation.md b/doc/documentation.md index 20ee566..a11bcfb 100644 --- a/doc/documentation.md +++ b/doc/documentation.md @@ -10,6 +10,12 @@ Query an ASN description history service (https://github.com/CIRCL/ASN-Descripti ----- +#### [btc](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc.py) + +An expansion hover module to get a blockchain balance from a BTC address in MISP. + +----- + #### [circl_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py) diff --git a/doc/expansion/btc.json b/doc/expansion/btc.json new file mode 100644 index 0000000..3aeceab --- /dev/null +++ b/doc/expansion/btc.json @@ -0,0 +1,3 @@ +{ + "description": "An expansion hover module to get a blockchain balance from a BTC address in MISP." +} diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index 3bb7253..7d3c2ce 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -29,6 +29,7 @@ import fnmatch import argparse import re import datetime +import psutil import tornado.web import tornado.process @@ -241,7 +242,23 @@ def main(): service = [(r'/modules', ListModules), (r'/query', QueryModule)] application = tornado.web.Application(service) - application.listen(port, address=listen) + try: + application.listen(port, address=listen) + except Exception as e: + if e.errno == 98: + pids = psutil.pids() + for pid in pids: + p = psutil.Process(pid) + if p.name() == "misp-modules": + print("\n\n\n") + print(e) + print("\nmisp-modules is still running as PID: {}\n".format(pid)) + print("Please kill accordingly:") + print("sudo kill {}".format(pid)) + sys.exit(-1) + print(e) + print("misp-modules might still be running.") + log.info('MISP modules server started on {0} port {1}'.format(listen, port)) if args.t: log.info('MISP modules started in test-mode, quitting immediately.') diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index f1c6d7a..1534fda 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,3 +1,3 @@ from . import _vmray -__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query'] +__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query'] diff --git a/misp_modules/modules/expansion/btc_steroids.py b/misp_modules/modules/expansion/btc_steroids.py new file mode 100755 index 0000000..c4edaa3 --- /dev/null +++ b/misp_modules/modules/expansion/btc_steroids.py @@ -0,0 +1,186 @@ +import sys +import json +import requests +import time + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['btc'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': 'BTC expansion service to \ + get quick information from MISP attributes', + 'module-type': ['hover']} + +moduleconfig = [] + +blockchain_firstseen='https://blockchain.info/q/addressfirstseen/' +blockchain_balance='https://blockchain.info/q/addressbalance/' +blockchain_totalreceived='https://blockchain.info/q/getreceivedbyaddress/' +blockchain_all='https://blockchain.info/rawaddr/' +converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts=' +converter_rls = 'https://min-api.cryptocompare.com/stats/rate/limit' +result_text = "" +g_rate_limit = 300 +start_time = 0 +conversion_rates = {} + +def get_consumption(output=False): + req = requests.get(converter_rls) + jreq = req.json() + minute = str(jreq['Minute']['CallsLeft']['Histo']) + hour = str(jreq['Hour']['CallsLeft']['Histo']) + # Debug out for the console + print("Calls left this minute / hour: " + minute + " / " + hour) + return minute, hour + + +def convert(btc, timestamp): + global g_rate_limit + global start_time + global now + global conversion_rates + date = time.strftime('%Y-%m-%d', time.localtime(timestamp)) + # Lookup conversion rates in the cache: + if date in conversion_rates: + (usd, eur) = conversion_rates[date] + else: + # If not cached, we have to get the converion rates + # We have to be careful with rate limiting on the server side + if g_rate_limit == 300: + minute, hour = get_consumption() + g_rate_limit -= 1 + now = time.time() + delta = now - start_time + #print(g_rate_limit) + if g_rate_limit <= 10: + minute, hour = get_consumption(output=True) + if int(minute) <= 10: + #print(minute) + #get_consumption(output=True) + time.sleep(3) + else: + mprint(minute) + start_time = time.time() + g_rate_limit = int(minute) + try: + req = requests.get(converter+str(timestamp)) + jreq = req.json() + usd = jreq['BTC']['USD'] + eur = jreq['BTC']['EUR'] + # Since we have the rates, store them in the cache + conversion_rates[date] = (usd, eur) + except Exception as ex: + mprint(ex) + get_consumption(output=True) + # Actually convert and return the values + u = usd * btc + e = eur * btc + return u,e + + +def mprint(input): + # Prepare the final print + global result_text + result_text = result_text + "\n" + str(input) + + +def handler(q=False): + global result_text + global conversion_rates + start_time = time.time() + now = time.time() + if q is False: + return False + request = json.loads(q) + click = False + # This means the magnifying glass has been clicked + if request.get('persistent') == 1: + click = True + # Otherwise the attribute was only hovered over + if request.get('btc'): + btc = request['btc'] + else: + return False + + mprint("\nAddress:\t" + btc) + try: + req = requests.get(blockchain_all+btc+"?limit=50&filter=5") + jreq = req.json() + except Exception as e: + #print(e) + print(req.text) + result_text = "" + sys.exit(1) + + n_tx = jreq['n_tx'] + balance = float(jreq['final_balance'] / 100000000) + rcvd = float(jreq['total_received'] / 100000000) + sent = float(jreq['total_sent'] / 100000000) + output = 'Balance:\t{0:.10f} BTC (+{1:.10f} BTC / -{2:.10f} BTC)' + mprint(output.format(balance, rcvd, sent)) + if click is False: + mprint("Transactions:\t" + str(n_tx) + "\t (previewing up to 5 most recent)") + else: + mprint("Transactions:\t" + str(n_tx)) + mprint("======================================================================================") + i = 0 + while i < n_tx: + if click is False: + req = requests.get(blockchain_all+btc+"?limit=5&offset="+str(i)+"&filter=5") + if n_tx > 5: + n_tx = 5 + else: + req = requests.get(blockchain_all+btc+"?limit=50&offset="+str(i)+"&filter=5") + jreq = req.json() + if jreq['txs']: + for transactions in jreq['txs']: + sum = 0 + sum_counter = 0 + for tx in transactions['inputs']: + script_old = tx['script'] + if tx['prev_out']['value'] != 0 and tx['prev_out']['addr'] == btc: + datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time']))) + value = float(tx['prev_out']['value'] / 100000000 ) + u,e = convert(value, transactions['time']) + mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0')) + if script_old != tx['script']: + i += 1 + else: + sum_counter += 1 + sum += value + if sum_counter > 1: + u,e = convert(sum, transactions['time']) + mprint("\t\t\t\t\t----------------------------------------------") + mprint("#" + str(n_tx - i) + "\t\t\t\t Sum:\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR\n".format(sum, u, e).rstrip('0')) + for tx in transactions['out']: + if tx['value'] != 0 and tx['addr'] == btc: + datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time']))) + value = float(tx['value'] / 100000000 ) + u,e = convert(value, transactions['time']) + mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t {0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0')) + #i += 1 + i += 1 + + r = { + 'results': [ + { + 'types': ['text'], + 'values':[ + str(result_text) + ] + } + ] + } + # Debug output on the console + print(result_text) + # Unset the result for the next request + result_text = "" + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 86abe7a..c9bca0e 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -65,16 +65,16 @@ def handle_expansion(api, ip, misperrors): for r in result['results']: if r['@category'] == 'pastries': - if r['@type'] == 'pastebin': + if r['source'] == 'pastebin': urls_pasties.append('https://pastebin.com/raw/%s' % r['key']) elif r['@category'] == 'synscan': asn_list.append(r['asn']) os_target = r['os'] if os_target != 'Unknown': os_list.append(r['os']) - elif r['@category'] == 'resolver' and r['@type'] =='reverse': + elif r['@category'] == 'resolver' and r['type'] =='reverse': domains_resolver.append(r['reverse']) - elif r['@category'] == 'resolver' and r['@type'] =='forward': + elif r['@category'] == 'resolver' and r['type'] =='forward': domains_forward.append(r['forward']) result_filtered['results'].append({'types': ['url'], 'values': urls_pasties, @@ -105,4 +105,4 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig - return moduleinfo \ No newline at end of file + return moduleinfo diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 7a05d12..3d6ef8e 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -315,7 +315,7 @@ def expand_pastries(api, misperror, **kwargs): status_ok = True for item in result['results']: if item['@category'] == 'pastries': - if item['@type'] == 'pastebin': + if item['source'] == 'pastebin': urls_pasties.append('https://pastebin.com/raw/%s' % item['key']) if 'domain' in item: @@ -374,4 +374,4 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig - return moduleinfo \ No newline at end of file + return moduleinfo diff --git a/tests/bodyhashdd.json b/tests/bodyhashdd.json index b6d256c..3bdfa82 100644 --- a/tests/bodyhashdd.json +++ b/tests/bodyhashdd.json @@ -1 +1 @@ -{"module": "hashdd", "md5": "838DE99E82C5B9753BAC96D82C1A8DCB"} +{"module": "hashdd", "md5": "838DE99E82C5B9753BAC96D82C1A8DCC"}