Merge branch 'master' of github.com:MISP/misp-modules into chrisr3d_patch

pull/253/head
chrisr3d 4 years ago
commit ad1ccdb9d0
  1. 1
      README.md
  2. 6
      REQUIREMENTS
  3. 6
      doc/documentation.md
  4. 3
      doc/expansion/btc.json
  5. 19
      misp_modules/__init__.py
  6. 2
      misp_modules/modules/expansion/__init__.py
  7. 186
      misp_modules/modules/expansion/btc_steroids.py
  8. 8
      misp_modules/modules/expansion/onyphe.py
  9. 4
      misp_modules/modules/expansion/onyphe_full.py
  10. 2
      tests/bodyhashdd.json

@ -18,6 +18,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
### Expansion modules
* [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
* [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP.
* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
* [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.

@ -21,8 +21,10 @@ domaintools_api
pygeoip
bs4
oauth2
yara
yara-python
sigmatools
stix2-patterns
maclookup
vulners
vulners
psutil
blockchain

@ -10,6 +10,12 @@ Query an ASN description history service (https://github.com/CIRCL/ASN-Descripti
-----
#### [btc](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc.py)
An expansion hover module to get a blockchain balance from a BTC address in MISP.
-----
#### [circl_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py)
<img src=logos/passivedns.png height=60>

@ -0,0 +1,3 @@
{
"description": "An expansion hover module to get a blockchain balance from a BTC address in MISP."
}

@ -29,6 +29,7 @@ import fnmatch
import argparse
import re
import datetime
import psutil
import tornado.web
import tornado.process
@ -241,7 +242,23 @@ def main():
service = [(r'/modules', ListModules), (r'/query', QueryModule)]
application = tornado.web.Application(service)
application.listen(port, address=listen)
try:
application.listen(port, address=listen)
except Exception as e:
if e.errno == 98:
pids = psutil.pids()
for pid in pids:
p = psutil.Process(pid)
if p.name() == "misp-modules":
print("\n\n\n")
print(e)
print("\nmisp-modules is still running as PID: {}\n".format(pid))
print("Please kill accordingly:")
print("sudo kill {}".format(pid))
sys.exit(-1)
print(e)
print("misp-modules might still be running.")
log.info('MISP modules server started on {0} port {1}'.format(listen, port))
if args.t:
log.info('MISP modules started in test-mode, quitting immediately.')

@ -1,3 +1,3 @@
from . import _vmray
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query']
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query']

@ -0,0 +1,186 @@
import sys
import json
import requests
import time
misperrors = {'error': 'Error'}
mispattributes = {'input': ['btc'], 'output': ['text']}
moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
'description': 'BTC expansion service to \
get quick information from MISP attributes',
'module-type': ['hover']}
moduleconfig = []
blockchain_firstseen='https://blockchain.info/q/addressfirstseen/'
blockchain_balance='https://blockchain.info/q/addressbalance/'
blockchain_totalreceived='https://blockchain.info/q/getreceivedbyaddress/'
blockchain_all='https://blockchain.info/rawaddr/'
converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts='
converter_rls = 'https://min-api.cryptocompare.com/stats/rate/limit'
result_text = ""
g_rate_limit = 300
start_time = 0
conversion_rates = {}
def get_consumption(output=False):
req = requests.get(converter_rls)
jreq = req.json()
minute = str(jreq['Minute']['CallsLeft']['Histo'])
hour = str(jreq['Hour']['CallsLeft']['Histo'])
# Debug out for the console
print("Calls left this minute / hour: " + minute + " / " + hour)
return minute, hour
def convert(btc, timestamp):
global g_rate_limit
global start_time
global now
global conversion_rates
date = time.strftime('%Y-%m-%d', time.localtime(timestamp))
# Lookup conversion rates in the cache:
if date in conversion_rates:
(usd, eur) = conversion_rates[date]
else:
# If not cached, we have to get the converion rates
# We have to be careful with rate limiting on the server side
if g_rate_limit == 300:
minute, hour = get_consumption()
g_rate_limit -= 1
now = time.time()
delta = now - start_time
#print(g_rate_limit)
if g_rate_limit <= 10:
minute, hour = get_consumption(output=True)
if int(minute) <= 10:
#print(minute)
#get_consumption(output=True)
time.sleep(3)
else:
mprint(minute)
start_time = time.time()
g_rate_limit = int(minute)
try:
req = requests.get(converter+str(timestamp))
jreq = req.json()
usd = jreq['BTC']['USD']
eur = jreq['BTC']['EUR']
# Since we have the rates, store them in the cache
conversion_rates[date] = (usd, eur)
except Exception as ex:
mprint(ex)
get_consumption(output=True)
# Actually convert and return the values
u = usd * btc
e = eur * btc
return u,e
def mprint(input):
# Prepare the final print
global result_text
result_text = result_text + "\n" + str(input)
def handler(q=False):
global result_text
global conversion_rates
start_time = time.time()
now = time.time()
if q is False:
return False
request = json.loads(q)
click = False
# This means the magnifying glass has been clicked
if request.get('persistent') == 1:
click = True
# Otherwise the attribute was only hovered over
if request.get('btc'):
btc = request['btc']
else:
return False
mprint("\nAddress:\t" + btc)
try:
req = requests.get(blockchain_all+btc+"?limit=50&filter=5")
jreq = req.json()
except Exception as e:
#print(e)
print(req.text)
result_text = ""
sys.exit(1)
n_tx = jreq['n_tx']
balance = float(jreq['final_balance'] / 100000000)
rcvd = float(jreq['total_received'] / 100000000)
sent = float(jreq['total_sent'] / 100000000)
output = 'Balance:\t{0:.10f} BTC (+{1:.10f} BTC / -{2:.10f} BTC)'
mprint(output.format(balance, rcvd, sent))
if click is False:
mprint("Transactions:\t" + str(n_tx) + "\t (previewing up to 5 most recent)")
else:
mprint("Transactions:\t" + str(n_tx))
mprint("======================================================================================")
i = 0
while i < n_tx:
if click is False:
req = requests.get(blockchain_all+btc+"?limit=5&offset="+str(i)+"&filter=5")
if n_tx > 5:
n_tx = 5
else:
req = requests.get(blockchain_all+btc+"?limit=50&offset="+str(i)+"&filter=5")
jreq = req.json()
if jreq['txs']:
for transactions in jreq['txs']:
sum = 0
sum_counter = 0
for tx in transactions['inputs']:
script_old = tx['script']
if tx['prev_out']['value'] != 0 and tx['prev_out']['addr'] == btc:
datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time'])))
value = float(tx['prev_out']['value'] / 100000000 )
u,e = convert(value, transactions['time'])
mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0'))
if script_old != tx['script']:
i += 1
else:
sum_counter += 1
sum += value
if sum_counter > 1:
u,e = convert(sum, transactions['time'])
mprint("\t\t\t\t\t----------------------------------------------")
mprint("#" + str(n_tx - i) + "\t\t\t\t Sum:\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR\n".format(sum, u, e).rstrip('0'))
for tx in transactions['out']:
if tx['value'] != 0 and tx['addr'] == btc:
datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time'])))
value = float(tx['value'] / 100000000 )
u,e = convert(value, transactions['time'])
mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t {0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0'))
#i += 1
i += 1
r = {
'results': [
{
'types': ['text'],
'values':[
str(result_text)
]
}
]
}
# Debug output on the console
print(result_text)
# Unset the result for the next request
result_text = ""
return r
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

@ -65,16 +65,16 @@ def handle_expansion(api, ip, misperrors):
for r in result['results']:
if r['@category'] == 'pastries':
if r['@type'] == 'pastebin':
if r['source'] == 'pastebin':
urls_pasties.append('https://pastebin.com/raw/%s' % r['key'])
elif r['@category'] == 'synscan':
asn_list.append(r['asn'])
os_target = r['os']
if os_target != 'Unknown':
os_list.append(r['os'])
elif r['@category'] == 'resolver' and r['@type'] =='reverse':
elif r['@category'] == 'resolver' and r['type'] =='reverse':
domains_resolver.append(r['reverse'])
elif r['@category'] == 'resolver' and r['@type'] =='forward':
elif r['@category'] == 'resolver' and r['type'] =='forward':
domains_forward.append(r['forward'])
result_filtered['results'].append({'types': ['url'], 'values': urls_pasties,
@ -105,4 +105,4 @@ def introspection():
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo
return moduleinfo

@ -315,7 +315,7 @@ def expand_pastries(api, misperror, **kwargs):
status_ok = True
for item in result['results']:
if item['@category'] == 'pastries':
if item['@type'] == 'pastebin':
if item['source'] == 'pastebin':
urls_pasties.append('https://pastebin.com/raw/%s' % item['key'])
if 'domain' in item:
@ -374,4 +374,4 @@ def introspection():
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo
return moduleinfo

@ -1 +1 @@
{"module": "hashdd", "md5": "838DE99E82C5B9753BAC96D82C1A8DCB"}
{"module": "hashdd", "md5": "838DE99E82C5B9753BAC96D82C1A8DCC"}

Loading…
Cancel
Save