MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

fix: Make pep8 happy

pipenv
Raphaël Vinot 2018-12-11 15:29:09 +01:00
parent d0aec62f1a
commit 8fc5b1fd1f
56 changed files with 695 additions and 638 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -13,7 +13,7 @@ python:
- "3.7-dev" - "3.7-dev"
install: install:
- pip install -U nose codecov pytest - pip install -U nose codecov pytest flake8
- pip install -U -r REQUIREMENTS - pip install -U -r REQUIREMENTS
- pip install . - pip install .
@ -30,6 +30,7 @@ script:
- sleep 5 - sleep 5
- nosetests --with-coverage --cover-package=misp_modules - nosetests --with-coverage --cover-package=misp_modules
- kill -s INT $pid - kill -s INT $pid
- flake8 --ignore=E501,W503 misp_modules
after_success: after_success:
- coverage combine .coverage* - coverage combine .coverage*

9
misp_modules/__init__.py
View File

@ -38,14 +38,14 @@ from tornado.concurrent import run_on_executor
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
try: try:
from .modules import * from .modules import * # noqa
HAS_PACKAGE_MODULES = True HAS_PACKAGE_MODULES = True
except Exception as e: except Exception as e:
print(e) print(e)
HAS_PACKAGE_MODULES = False HAS_PACKAGE_MODULES = False
try: try:
from .helpers import * from .helpers import * # noqa
HAS_PACKAGE_HELPERS = True HAS_PACKAGE_HELPERS = True
except Exception as e: except Exception as e:
print(e) print(e)
@ -148,7 +148,7 @@ def load_package_modules():
mhandlers = {} mhandlers = {}
modules = [] modules = []
for path, module in sys.modules.items(): for path, module in sys.modules.items():
r = re.findall("misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path) r = re.findall(r"misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
if r and len(r[0]) == 2: if r and len(r[0]) == 2:
moduletype, modulename = r[0] moduletype, modulename = r[0]
mhandlers[modulename] = module mhandlers[modulename] = module
@ -159,6 +159,9 @@ def load_package_modules():
class ListModules(tornado.web.RequestHandler): class ListModules(tornado.web.RequestHandler):
global loaded_modules
global mhandlers
def get(self): def get(self):
ret = [] ret = []
for module in loaded_modules: for module in loaded_modules:

3
misp_modules/helpers/cache.py
View File

@ -33,7 +33,7 @@ def selftest(enable=True):
r = redis.StrictRedis(host=hostname, port=port, db=db) r = redis.StrictRedis(host=hostname, port=port, db=db)
try: try:
r.ping() r.ping()
except: except Exception:
return 'Redis not running or not installed. Helper will be disabled.' return 'Redis not running or not installed. Helper will be disabled.'
@ -62,6 +62,7 @@ def flush():
returncode = r.flushdb() returncode = r.flushdb()
return returncode return returncode
if __name__ == "__main__": if __name__ == "__main__":
import sys import sys
if selftest() is not None: if selftest() is not None:

6
misp_modules/modules/__init__.py
View File

@ -1,3 +1,3 @@
from .expansion import * from .expansion import * # noqa
from .import_mod import * from .import_mod import * # noqa
from .export_mod import * from .export_mod import * # noqa

2
misp_modules/modules/expansion/__init__.py
View File

@ -1,4 +1,4 @@
from . import _vmray from . import _vmray # noqa
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', __all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl',
'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi',

59
misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py
View File

@ -47,9 +47,11 @@ options = None
locale.setlocale(locale.LC_ALL, '') locale.setlocale(locale.LC_ALL, '')
class QueryError(Exception): class QueryError(Exception):
pass pass
class DnsdbClient(object): class DnsdbClient(object):
def __init__(self, server, apikey, limit=None, http_proxy=None, https_proxy=None): def __init__(self, server, apikey, limit=None, http_proxy=None, https_proxy=None):
self.server = server self.server = server
@ -81,7 +83,6 @@ class DnsdbClient(object):
return self._query(path, before, after) return self._query(path, before, after)
def _query(self, path, before=None, after=None): def _query(self, path, before=None, after=None):
res = []
url = '%s/lookup/%s' % (self.server, path) url = '%s/lookup/%s' % (self.server, path)
params = {} params = {}
@ -120,12 +121,15 @@ class DnsdbClient(object):
except (HTTPError, URLError) as e: except (HTTPError, URLError) as e:
raise QueryError(str(e), sys.exc_traceback) raise QueryError(str(e), sys.exc_traceback)
def quote(path): def quote(path):
return urllib_quote(path, safe='') return urllib_quote(path, safe='')
def sec_to_text(ts): def sec_to_text(ts):
return time.strftime('%Y-%m-%d %H:%M:%S -0000', time.gmtime(ts)) return time.strftime('%Y-%m-%d %H:%M:%S -0000', time.gmtime(ts))
def rrset_to_text(m): def rrset_to_text(m):
s = StringIO() s = StringIO()
@ -155,9 +159,11 @@ def rrset_to_text(m):
finally: finally:
s.close() s.close()
def rdata_to_text(m): def rdata_to_text(m):
return '%s IN %s %s' % (m['rrname'], m['rrtype'], m['rdata']) return '%s IN %s %s' % (m['rrname'], m['rrtype'], m['rdata'])
def parse_config(cfg_files): def parse_config(cfg_files):
config = {} config = {}
@ -172,6 +178,7 @@ def parse_config(cfg_files):
return config return config
def time_parse(s): def time_parse(s):
try: try:
epoch = int(s) epoch = int(s)
@ -193,14 +200,15 @@ def time_parse(s):
m = re.match(r'^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$', s, re.I) m = re.match(r'^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$', s, re.I)
if m: if m:
return -1*(int(m.group(1) or 0)*604800 + return -1 * (int(m.group(1) or 0) * 604800
int(m.group(2) or 0)*86400+ + int(m.group(2) or 0) * 86400
int(m.group(3) or 0)*3600+ + int(m.group(3) or 0) * 3600
int(m.group(4) or 0)*60+ + int(m.group(4) or 0) * 60
int(m.group(5) or 0)) + int(m.group(5) or 0))
raise ValueError('Invalid time: "%s"' % s) raise ValueError('Invalid time: "%s"' % s)
def epipe_wrapper(func): def epipe_wrapper(func):
def f(*args, **kwargs): def f(*args, **kwargs):
try: try:
@ -211,31 +219,23 @@ def epipe_wrapper(func):
raise raise
return f return f
@epipe_wrapper @epipe_wrapper
def main(): def main():
global cfg global cfg
global options global options
parser = optparse.OptionParser(epilog='Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)') parser = optparse.OptionParser(epilog='Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)')
parser.add_option('-c', '--config', dest='config', parser.add_option('-c', '--config', dest='config', help='config file', action='append')
help='config file', action='append') parser.add_option('-r', '--rrset', dest='rrset', type='string', help='rrset <ONAME>[/<RRTYPE>[/BAILIWICK]]')
parser.add_option('-r', '--rrset', dest='rrset', type='string', parser.add_option('-n', '--rdataname', dest='rdata_name', type='string', help='rdata name <NAME>[/<RRTYPE>]')
help='rrset <ONAME>[/<RRTYPE>[/BAILIWICK]]') parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string', help='rdata ip <IPADDRESS|IPRANGE|IPNETWORK>')
parser.add_option('-n', '--rdataname', dest='rdata_name', type='string', parser.add_option('-t', '--rrtype', dest='rrtype', type='string', help='rrset or rdata rrtype')
help='rdata name <NAME>[/<RRTYPE>]') parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string', help='rrset bailiwick')
parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string',
help='rdata ip <IPADDRESS|IPRANGE|IPNETWORK>')
parser.add_option('-t', '--rrtype', dest='rrtype', type='string',
help='rrset or rdata rrtype')
parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string',
help='rrset bailiwick')
parser.add_option('-s', '--sort', dest='sort', type='string', help='sort key') parser.add_option('-s', '--sort', dest='sort', type='string', help='sort key')
parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False, parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False, help='reverse sort')
help='reverse sort') parser.add_option('-j', '--json', dest='json', action='store_true', default=False, help='output in JSON format')
parser.add_option('-j', '--json', dest='json', action='store_true', default=False, parser.add_option('-l', '--limit', dest='limit', type='int', default=0, help='limit number of results')
help='output in JSON format')
parser.add_option('-l', '--limit', dest='limit', type='int', default=0,
help='limit number of results')
parser.add_option('', '--before', dest='before', type='string', help='only output results seen before this time') parser.add_option('', '--before', dest='before', type='string', help='only output results seen before this time')
parser.add_option('', '--after', dest='after', type='string', help='only output results seen after this time') parser.add_option('', '--after', dest='after', type='string', help='only output results seen after this time')
@ -263,13 +263,13 @@ def main():
print(str(e), file=sys.stderr) print(str(e), file=sys.stderr)
sys.exit(1) sys.exit(1)
if not 'DNSDB_SERVER' in cfg: if 'DNSDB_SERVER' not in cfg:
cfg['DNSDB_SERVER'] = DEFAULT_DNSDB_SERVER cfg['DNSDB_SERVER'] = DEFAULT_DNSDB_SERVER
if not 'HTTP_PROXY' in cfg: if 'HTTP_PROXY' not in cfg:
cfg['HTTP_PROXY'] = DEFAULT_HTTP_PROXY cfg['HTTP_PROXY'] = DEFAULT_HTTP_PROXY
if not 'HTTPS_PROXY' in cfg: if 'HTTPS_PROXY' not in cfg:
cfg['HTTPS_PROXY'] = DEFAULT_HTTPS_PROXY cfg['HTTPS_PROXY'] = DEFAULT_HTTPS_PROXY
if not 'APIKEY' in cfg: if 'APIKEY' not in cfg:
sys.stderr.write('dnsdb_query: APIKEY not defined in config file\n') sys.stderr.write('dnsdb_query: APIKEY not defined in config file\n')
sys.exit(1) sys.exit(1)
@ -307,7 +307,7 @@ def main():
if options.sort: if options.sort:
results = list(results) results = list(results)
if len(results) > 0: if len(results) > 0:
if not options.sort in results[0]: if options.sort not in results[0]:
sort_keys = results[0].keys() sort_keys = results[0].keys()
sort_keys.sort() sort_keys.sort()
sys.stderr.write('dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' % (options.sort, ', '.join(sort_keys))) sys.stderr.write('dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' % (options.sort, ', '.join(sort_keys)))
@ -319,5 +319,6 @@ def main():
print(e.message, file=sys.stderr) print(e.message, file=sys.stderr)
sys.exit(1) sys.exit(1)
if __name__ == '__main__': if __name__ == '__main__':
main() main()

6
misp_modules/modules/expansion/asn_history.py
View File

@ -22,9 +22,9 @@ def handler(q=False):
misperrors['error'] = "Unsupported attributes type" misperrors['error'] = "Unsupported attributes type"
return misperrors return misperrors
if not request.get('config') and not (request['config'].get('host') and if not request.get('config') and not (request['config'].get('host')
request['config'].get('port') and and request['config'].get('port')
request['config'].get('db')): and request['config'].get('db')):
misperrors['error'] = 'ASN description history configuration is missing' misperrors['error'] = 'ASN description history configuration is missing'
return misperrors return misperrors

23
misp_modules/modules/expansion/btc_steroids.py
View File

@ -15,21 +15,22 @@ moduleconfig = []
blockchain_firstseen = 'https://blockchain.info/q/addressfirstseen/' blockchain_firstseen = 'https://blockchain.info/q/addressfirstseen/'
blockchain_balance = 'https://blockchain.info/q/addressbalance/' blockchain_balance = 'https://blockchain.info/q/addressbalance/'
blockchain_totalreceived = 'https://blockchain.info/q/getreceivedbyaddress/' blockchain_totalreceived = 'https://blockchain.info/q/getreceivedbyaddress/'
blockchain_all='https://blockchain.info/rawaddr/' blockchain_all = 'https://blockchain.info/rawaddr/{}?filter=5{}'
converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts=' converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts={}'
converter_rls = 'https://min-api.cryptocompare.com/stats/rate/limit' converter_rls = 'https://min-api.cryptocompare.com/stats/rate/limit'
result_text = "" result_text = ""
g_rate_limit = 300 g_rate_limit = 300
start_time = 0 start_time = 0
conversion_rates = {} conversion_rates = {}
def get_consumption(output=False): def get_consumption(output=False):
try: try:
req = requests.get(converter_rls) req = requests.get(converter_rls)
jreq = req.json() jreq = req.json()
minute = str(jreq['Data']['calls_left']['minute']) minute = str(jreq['Data']['calls_left']['minute'])
hour = str(jreq['Data']['calls_left']['hour']) hour = str(jreq['Data']['calls_left']['hour'])
except: except Exception:
minute = str(-1) minute = str(-1)
hour = str(-1) hour = str(-1)
# Debug out for the console # Debug out for the console
@ -53,7 +54,7 @@ def convert(btc, timestamp):
minute, hour = get_consumption() minute, hour = get_consumption()
g_rate_limit -= 1 g_rate_limit -= 1
now = time.time() now = time.time()
delta = now - start_time # delta = now - start_time
# print(g_rate_limit) # print(g_rate_limit)
if g_rate_limit <= 10: if g_rate_limit <= 10:
minute, hour = get_consumption(output=True) minute, hour = get_consumption(output=True)
@ -66,7 +67,7 @@ def convert(btc, timestamp):
start_time = time.time() start_time = time.time()
g_rate_limit = int(minute) g_rate_limit = int(minute)
try: try:
req = requests.get(converter+str(timestamp)) req = requests.get(converter.format(timestamp))
jreq = req.json() jreq = req.json()
usd = jreq['BTC']['USD'] usd = jreq['BTC']['USD']
eur = jreq['BTC']['EUR'] eur = jreq['BTC']['EUR']
@ -90,8 +91,8 @@ def mprint(input):
def handler(q=False): def handler(q=False):
global result_text global result_text
global conversion_rates global conversion_rates
start_time = time.time() # start_time = time.time()
now = time.time() # now = time.time()
if q is False: if q is False:
return False return False
request = json.loads(q) request = json.loads(q)
@ -107,9 +108,9 @@ def handler(q=False):
mprint("\nAddress:\t" + btc) mprint("\nAddress:\t" + btc)
try: try:
req = requests.get(blockchain_all+btc+"?limit=50&filter=5") req = requests.get(blockchain_all.format(btc, "&limit=50"))
jreq = req.json() jreq = req.json()
except Exception as e: except Exception:
# print(e) # print(e)
print(req.text) print(req.text)
result_text = "" result_text = ""
@ -130,11 +131,11 @@ def handler(q=False):
i = 0 i = 0
while i < n_tx: while i < n_tx:
if click is False: if click is False:
req = requests.get(blockchain_all+btc+"?limit=5&offset="+str(i)+"&filter=5") req = requests.get(blockchain_all.format(btc, "&limit=5&offset={}".format(i)))
if n_tx > 5: if n_tx > 5:
n_tx = 5 n_tx = 5
else: else:
req = requests.get(blockchain_all+btc+"?limit=50&offset="+str(i)+"&filter=5") req = requests.get(blockchain_all.format(btc, "&limit=50&offset={}".format(i)))
jreq = req.json() jreq = req.json()
if jreq['txs']: if jreq['txs']:
for transactions in jreq['txs']: for transactions in jreq['txs']:

2
misp_modules/modules/expansion/countrycode.py
View File

@ -22,6 +22,7 @@ common_tlds = {"com":"Commercial (Worldwide)",
codes = False codes = False
def handler(q=False): def handler(q=False):
global codes global codes
if not codes: if not codes:
@ -58,4 +59,3 @@ def introspection():
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

7
misp_modules/modules/expansion/dbl_spamhaus.py
View File

@ -1,6 +1,5 @@
import json import json
import datetime import sys
from collections import defaultdict
try: try:
import dns.resolver import dns.resolver
@ -30,12 +29,14 @@ dbl_mapping = {'127.0.1.2': 'spam domain',
'127.0.1.106': 'abused legit botnet C&C', '127.0.1.106': 'abused legit botnet C&C',
'127.0.1.255': 'IP queries prohibited!'} '127.0.1.255': 'IP queries prohibited!'}
def fetch_requested_value(request): def fetch_requested_value(request):
for attribute_type in mispattributes['input']: for attribute_type in mispattributes['input']:
if request.get(attribute_type): if request.get(attribute_type):
return request[attribute_type].split('|')[0] return request[attribute_type].split('|')[0]
return None return None
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -52,9 +53,11 @@ def handler(q=False):
result = str(e) result = str(e)
return {'results': [{'types': mispattributes.get('output'), 'values': result}]} return {'results': [{'types': mispattributes.get('output'), 'values': result}]}
def introspection(): def introspection():
return mispattributes return mispattributes
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

2
misp_modules/modules/expansion/dns.py
View File

@ -43,7 +43,7 @@ def handler(q=False):
except dns.exception.Timeout: except dns.exception.Timeout:
misperrors['error'] = "Timeout" misperrors['error'] = "Timeout"
return misperrors return misperrors
except: except Exception:
misperrors['error'] = "DNS resolving error" misperrors['error'] = "DNS resolving error"
return misperrors return misperrors

6
misp_modules/modules/expansion/farsight_passivedns.py
View File

@ -51,7 +51,7 @@ def lookup_name(client, name):
for i in item.get('rdata'): for i in item.get('rdata'):
# grab email field and replace first dot by @ to convert to an email address # grab email field and replace first dot by @ to convert to an email address
yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1)) yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1))
except QueryError as e: except QueryError:
pass pass
try: try:
@ -59,7 +59,7 @@ def lookup_name(client, name):
for item in res: for item in res:
if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: if item.get('rrtype') in ['A', 'AAAA', 'CNAME']:
yield(item.get('rrname').rstrip('.')) yield(item.get('rrname').rstrip('.'))
except QueryError as e: except QueryError:
pass pass
@ -68,7 +68,7 @@ def lookup_ip(client, ip):
res = client.query_rdata_ip(ip) res = client.query_rdata_ip(ip)
for item in res: for item in res:
yield(item['rrname'].rstrip('.')) yield(item['rrname'].rstrip('.'))
except QueryError as e: except QueryError:
pass pass

4
misp_modules/modules/expansion/geoip_country.py
View File

@ -27,7 +27,7 @@ try:
config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'geoip_country.cfg')) config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'geoip_country.cfg'))
gi = pygeoip.GeoIP(config.get('GEOIP', 'database')) gi = pygeoip.GeoIP(config.get('GEOIP', 'database'))
enabled = True enabled = True
except: except Exception:
enabled = False enabled = False
@ -49,7 +49,7 @@ def handler(q=False):
try: try:
answer = gi.country_code_by_addr(toquery) answer = gi.country_code_by_addr(toquery)
except: except Exception:
misperrors['error'] = "GeoIP resolving error" misperrors['error'] = "GeoIP resolving error"
return misperrors return misperrors

6
misp_modules/modules/expansion/ipasn.py
View File

@ -24,9 +24,9 @@ def handler(q=False):
misperrors['error'] = "Unsupported attributes type" misperrors['error'] = "Unsupported attributes type"
return misperrors return misperrors
if not request.get('config') and not (request['config'].get('host') and if not request.get('config') and not (request['config'].get('host')
request['config'].get('port') and and request['config'].get('port')
request['config'].get('db')): and request['config'].get('db')):
misperrors['error'] = 'IP ASN history configuration is missing' misperrors['error'] = 'IP ASN history configuration is missing'
return misperrors return misperrors

4
misp_modules/modules/expansion/iprep.py
View File

@ -45,7 +45,7 @@ def parse_iprep(ip, api):
url = 'https://www.packetmail.net/iprep.php/%s' % ip url = 'https://www.packetmail.net/iprep.php/%s' % ip
try: try:
data = requests.get(url, params={'apikey': api}).json() data = requests.get(url, params={'apikey': api}).json()
except: except Exception:
return ['Error pulling data'], rep return ['Error pulling data'], rep
# print '%s' % data # print '%s' % data
for name, val in data.items(): for name, val in data.items():
@ -72,7 +72,7 @@ def parse_iprep(ip, api):
full_text += '\n%s' % context full_text += '\n%s' % context
misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen']) misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen'])
rep.append({'types': mispattributes['output'], 'categories': ['External analysis'], 'values': misp_val, 'comment': misp_comment}) rep.append({'types': mispattributes['output'], 'categories': ['External analysis'], 'values': misp_val, 'comment': misp_comment})
except: except Exception:
err.append('Error parsing source: %s' % name) err.append('Error parsing source: %s' % name)
rep.append({'types': ['freetext'], 'values': full_text, 'comment': 'Free text import of IPRep'}) rep.append({'types': ['freetext'], 'values': full_text, 'comment': 'Free text import of IPRep'})

8
misp_modules/modules/expansion/macaddress_io.py
View File

@ -86,8 +86,8 @@ def handler(q=False):
response.block_details.date_updated.strftime('%d %B %Y') if response.block_details.date_updated else None response.block_details.date_updated.strftime('%d %B %Y') if response.block_details.date_updated else None
results = { results = {
'results': [ 'results':
{'types': ['text'], 'values': [{'types': ['text'], 'values':
{ {
# Mac address details # Mac address details
'Valid MAC address': "True" if response.mac_address_details.is_valid else "False", 'Valid MAC address': "True" if response.mac_address_details.is_valid else "False",
@ -109,9 +109,7 @@ def handler(q=False):
'Assignment block size': response.block_details.assignment_block_size, 'Assignment block size': response.block_details.assignment_block_size,
'Date when the range was allocated': date_created, 'Date when the range was allocated': date_created,
'Date when the range was last updated': date_updated 'Date when the range was last updated': date_updated
} }}]
}
]
} }
return results return results

6
misp_modules/modules/expansion/onyphe.py
View File

@ -1,4 +1,3 @@
import json
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import json import json
@ -9,7 +8,8 @@ except ImportError:
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], 'output': ['hostname', 'domain', 'ip-src', 'ip-dst','url']} mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']}
# possible module-types: 'expansion', 'hover' or both # possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
'description': 'Query on Onyphe', 'description': 'Query on Onyphe',
@ -54,7 +54,7 @@ def handle_expansion(api, ip, misperrors):
misperrors['error'] = result['message'] misperrors['error'] = result['message']
return misperrors return misperrors
categories = list(set([item['@category'] for item in result['results']])) # categories = list(set([item['@category'] for item in result['results']]))
result_filtered = {"results": []} result_filtered = {"results": []}
urls_pasties = [] urls_pasties = []

4
misp_modules/modules/expansion/onyphe_full.py
View File

@ -1,4 +1,3 @@
import json
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import json import json
@ -188,7 +187,7 @@ def expand_syscan(api, ip, misperror):
def expand_datascan(api, misperror, **kwargs): def expand_datascan(api, misperror, **kwargs):
status_ok = False status_ok = False
r = [] r = []
ip = '' # ip = ''
query = '' query = ''
asn_list = [] asn_list = []
geoloc = [] geoloc = []
@ -368,6 +367,7 @@ def expand_threatlist(api, misperror,**kwargs):
return r, status_ok return r, status_ok
def introspection(): def introspection():
return mispattributes return mispattributes

8
misp_modules/modules/expansion/otx.py
View File

@ -15,6 +15,7 @@ moduleinfo = {'version': '1', 'author': 'chrisdoman',
# We're not actually using the API key yet # We're not actually using the API key yet
moduleconfig = ["apikey"] moduleconfig = ["apikey"]
# Avoid adding windows update to enrichment etc. # Avoid adding windows update to enrichment etc.
def isBlacklisted(value): def isBlacklisted(value):
blacklist = ['0.0.0.0', '8.8.8.8', '255.255.255.255', '192.168.56.', 'time.windows.com'] blacklist = ['0.0.0.0', '8.8.8.8', '255.255.255.255', '192.168.56.', 'time.windows.com']
@ -25,10 +26,12 @@ def isBlacklisted(value):
return True return True
def valid_ip(ip): def valid_ip(ip):
m = re.match(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$", ip) m = re.match(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$", ip)
return bool(m) and all(map(lambda n: 0 <= int(n) <= 255, m.groups())) return bool(m) and all(map(lambda n: 0 <= int(n) <= 255, m.groups()))
def findAll(data, keys): def findAll(data, keys):
a = [] a = []
if isinstance(data, dict): if isinstance(data, dict):
@ -43,9 +46,11 @@ def findAll(data, keys):
a.extend(findAll(i, keys)) a.extend(findAll(i, keys))
return a return a
def valid_email(email): def valid_email(email):
return bool(re.search(r"[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?", email)) return bool(re.search(r"[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?", email))
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -104,14 +109,12 @@ def getIP(ip, key):
for _hash in findAll(req, "hash"): for _hash in findAll(req, "hash"):
ret.append({"types": ["sha256"], "values": [_hash]}) ret.append({"types": ["sha256"], "values": [_hash]})
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text) req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text)
for hostname in findAll(req, "hostname"): for hostname in findAll(req, "hostname"):
if not isBlacklisted(hostname): if not isBlacklisted(hostname):
ret.append({"types": ["hostname"], "values": [hostname]}) ret.append({"types": ["hostname"], "values": [hostname]})
return ret return ret
@ -144,6 +147,7 @@ def getDomain(domain, key):
return ret return ret
def introspection(): def introspection():
return mispattributes return mispattributes

2
misp_modules/modules/expansion/passivetotal.py
View File

@ -331,7 +331,7 @@ def handler(q=False):
output['results'] += results output['results'] += results
else: else:
log.error("Unsupported query pattern issued.") log.error("Unsupported query pattern issued.")
except: except Exception:
return misperrors return misperrors
return output return output

5
misp_modules/modules/expansion/rbl.py
View File

@ -1,5 +1,5 @@
import json import json
import datetime import sys
try: try:
import dns.resolver import dns.resolver
@ -76,6 +76,7 @@ rbls = {
'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html' 'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html'
} }
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -101,9 +102,11 @@ def handler(q=False):
result = "\n".join(["{}: {}".format(l, " - ".join(i)) for l, i in zip(listed, info)]) result = "\n".join(["{}: {}".format(l, " - ".join(i)) for l, i in zip(listed, info)])
return {'results': [{'types': mispattributes.get('output'), 'values': result}]} return {'results': [{'types': mispattributes.get('output'), 'values': result}]}
def introspection(): def introspection():
return mispattributes return mispattributes
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

15
misp_modules/modules/expansion/reversedns.py
View File

@ -1,5 +1,5 @@
import json import json
import dns.reversename, dns.resolver from dns import reversename, resolver, exception
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['hostname']} mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['hostname']}
@ -12,6 +12,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
# config fields that your code expects from the site admin # config fields that your code expects from the site admin
moduleconfig = ['nameserver'] moduleconfig = ['nameserver']
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -26,9 +27,9 @@ def handler(q=False):
return False return False
# reverse lookup for ip # reverse lookup for ip
revname = dns.reversename.from_address(toquery) revname = reversename.from_address(toquery)
r = dns.resolver.Resolver() r = resolver.Resolver()
r.timeout = 2 r.timeout = 2
r.lifetime = 2 r.lifetime = 2
@ -42,13 +43,13 @@ def handler(q=False):
try: try:
answer = r.query(revname, 'PTR') answer = r.query(revname, 'PTR')
except dns.resolver.NXDOMAIN: except resolver.NXDOMAIN:
misperrors['error'] = "NXDOMAIN" misperrors['error'] = "NXDOMAIN"
return misperrors return misperrors
except dns.exception.Timeout: except exception.Timeout:
misperrors['error'] = "Timeout" misperrors['error'] = "Timeout"
return misperrors return misperrors
except: except Exception:
misperrors['error'] = "DNS resolving error" misperrors['error'] = "DNS resolving error"
return misperrors return misperrors
@ -56,9 +57,11 @@ def handler(q=False):
'values':[str(answer[0])]}]} 'values':[str(answer[0])]}]}
return r return r
def introspection(): def introspection():
return mispattributes return mispattributes
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

6
misp_modules/modules/expansion/securitytrails.py
View File

@ -114,8 +114,7 @@ def handle_domain(api, domain, misperrors):
if r: if r:
result_filtered['results'].extend(r) result_filtered['results'].extend(r)
else: else:
misperrors['error'] = misperrors[ misperrors['error'] = misperrors['error'] + ' Error in expand History DNS'
'error'] + ' Error in expand History DNS'
return misperrors return misperrors
r, status_ok = expand_history_whois(api, domain) r, status_ok = expand_history_whois(api, domain)
@ -124,8 +123,7 @@ def handle_domain(api, domain, misperrors):
if r: if r:
result_filtered['results'].extend(r) result_filtered['results'].extend(r)
else: else:
misperrors['error'] = misperrors['error'] + \ misperrors['error'] = misperrors['error'] + ' Error in expand History Whois'
' Error in expand History Whois'
return misperrors return misperrors
return result_filtered return result_filtered

9
misp_modules/modules/expansion/sigma_queries.py
View File

@ -1,4 +1,6 @@
import sys, os, io, json import sys
import io
import json
try: try:
from sigma.parser import SigmaCollectionParser from sigma.parser import SigmaCollectionParser
from sigma.config import SigmaConfiguration from sigma.config import SigmaConfiguration
@ -13,6 +15,7 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['e
moduleconfig = [] moduleconfig = []
sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'wdatp', 'splunkxml', 'arcsight', 'qualys') sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'wdatp', 'splunkxml', 'arcsight', 'qualys')
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -35,16 +38,18 @@ def handler(q=False):
backend.finalize() backend.finalize()
print("#NEXT") print("#NEXT")
targets.append(t) targets.append(t)
except: except Exception:
continue continue
sys.stdout = old_stdout sys.stdout = old_stdout
results = result.getvalue()[:-5].split('#NEXT') results = result.getvalue()[:-5].split('#NEXT')
d_result = {t: r.strip() for t, r in zip(targets, results)} d_result = {t: r.strip() for t, r in zip(targets, results)}
return {'results': [{'types': mispattributes['output'], 'values': d_result}]} return {'results': [{'types': mispattributes['output'], 'values': d_result}]}
def introspection(): def introspection():
return mispattributes return mispattributes
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

3
misp_modules/modules/expansion/sigma_syntax_validator.py
View File

@ -12,6 +12,7 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['e
'description': 'An expansion hover module to perform a syntax check on sigma rules'} 'description': 'An expansion hover module to perform a syntax check on sigma rules'}
moduleconfig = [] moduleconfig = []
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -27,9 +28,11 @@ def handler(q=False):
result = ("Syntax error: {}".format(str(e))) result = ("Syntax error: {}".format(str(e)))
return {'results': [{'types': mispattributes['output'], 'values': result}]} return {'results': [{'types': mispattributes['output'], 'values': result}]}
def introspection(): def introspection():
return mispattributes return mispattributes
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

3
misp_modules/modules/expansion/stix2_pattern_syntax_validator.py
View File

@ -10,6 +10,7 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['e
'description': 'An expansion hover module to perform a syntax check on stix2 patterns.'} 'description': 'An expansion hover module to perform a syntax check on stix2 patterns.'}
moduleconfig = [] moduleconfig = []
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -34,9 +35,11 @@ def handler(q=False):
result = "Syntax valid" result = "Syntax valid"
return {'results': [{'types': mispattributes['output'], 'values': result}]} return {'results': [{'types': mispattributes['output'], 'values': result}]}
def introspection(): def introspection():
return mispattributes return mispattributes
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

12
misp_modules/modules/expansion/threatcrowd.py
View File

@ -25,28 +25,31 @@ def isBlacklisted(value):
return False return False
def valid_ip(ip): def valid_ip(ip):
m = re.match(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$", ip) m = re.match(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$", ip)
return bool(m) and all(map(lambda n: 0 <= int(n) <= 255, m.groups())) return bool(m) and all(map(lambda n: 0 <= int(n) <= 255, m.groups()))
def valid_domain(hostname): def valid_domain(hostname):
if len(hostname) > 255: if len(hostname) > 255:
return False return False
if hostname[-1] == ".": if hostname[-1] == ".":
hostname = hostname[:-1] # strip exactly one dot from the right, if present hostname = hostname[:-1] # strip exactly one dot from the right, if present
allowed = re.compile("(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE) allowed = re.compile(r"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
return all(allowed.match(x) for x in hostname.split(".")) return all(allowed.match(x) for x in hostname.split("."))
def valid_email(email): def valid_email(email):
return bool(re.search(r"^[\w\.\+\-]+\@[\w]+\.[a-z]{2,3}$", email)) return bool(re.search(r"^[\w\.\+\-]+\@[\w]+\.[a-z]{2,3}$", email))
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
q = json.loads(q) q = json.loads(q)
r = {"results": []} r = {"results": []}
if "ip-src" in q: if "ip-src" in q:
@ -110,11 +113,9 @@ def getIP(ip):
for hash in req["hashes"]: for hash in req["hashes"]:
ret.append({"types": ["md5"], "values": [hash]}) ret.append({"types": ["md5"], "values": [hash]})
return ret return ret
def getEmail(email): def getEmail(email):
ret = [] ret = []
j = requests.get("https://www.threatcrowd.org/searchApi/v2/email/report/?email=" + email).text j = requests.get("https://www.threatcrowd.org/searchApi/v2/email/report/?email=" + email).text
@ -129,7 +130,6 @@ def getEmail(email):
return ret return ret
def getDomain(domain): def getDomain(domain):
ret = [] ret = []
@ -148,9 +148,9 @@ def getDomain(domain):
for hash in req["hashes"]: for hash in req["hashes"]:
ret.append({"types": ["md5"], "values": [hash]}) ret.append({"types": ["md5"], "values": [hash]})
return ret return ret
def introspection(): def introspection():
return mispattributes return mispattributes

2
misp_modules/modules/expansion/threatminer.py
View File

@ -1,7 +1,5 @@
import json import json
import requests import requests
from requests import HTTPError
import base64
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'md5', 'sha1', 'sha256', 'sha512'], mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'md5', 'sha1', 'sha256', 'sha512'],

4
misp_modules/modules/expansion/virustotal.py
View File

@ -19,6 +19,7 @@ moduleconfig = ["apikey", "event_limit"]
comment = '{}: Enriched via VirusTotal' comment = '{}: Enriched via VirusTotal'
hash_types = ["md5", "sha1", "sha256", "sha512"] hash_types = ["md5", "sha1", "sha256", "sha512"]
class VirusTotalRequest(object): class VirusTotalRequest(object):
def __init__(self, config): def __init__(self, config):
self.apikey = config['apikey'] self.apikey = config['apikey']
@ -146,6 +147,7 @@ class VirusTotalRequest(object):
self.to_return.append({"types": ["malware-sample"], "categories": ["Payload delivery"], self.to_return.append({"types": ["malware-sample"], "categories": ["Payload delivery"],
"values": data["submimssion_names"], "data": str(base64.b64encore(malsample), 'utf-8')}) "values": data["submimssion_names"], "data": str(base64.b64encore(malsample), 'utf-8')})
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -161,9 +163,11 @@ def handler(q=False):
return misperrors return misperrors
return {'results': r} return {'results': r}
def introspection(): def introspection():
return mispattributes return mispattributes
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

6
misp_modules/modules/expansion/vmray_submit.py
View File

@ -55,7 +55,7 @@ def handler(q=False):
else: else:
misperrors['error'] = "No malware sample or attachment supplied" misperrors['error'] = "No malware sample or attachment supplied"
return misperrors return misperrors
except: except Exception:
misperrors['error'] = "Unable to process submited sample data" misperrors['error'] = "Unable to process submited sample data"
return misperrors return misperrors
@ -102,7 +102,7 @@ def handler(q=False):
return misperrors return misperrors
else: else:
return vmrayProcess(vmraydata) return vmrayProcess(vmraydata)
except: except Exception:
misperrors['error'] = "Problem when calling API." misperrors['error'] = "Problem when calling API."
return misperrors return misperrors
else: else:
@ -148,7 +148,7 @@ def vmrayProcess(vmraydata):
else: else:
misperrors['error'] = "No valid results returned." misperrors['error'] = "No valid results returned."
return misperrors return misperrors
except: except Exception:
misperrors['error'] = "No valid submission data returned." misperrors['error'] = "No valid submission data returned."
return misperrors return misperrors
else: else:

2
misp_modules/modules/expansion/vulndb.py
View File

@ -268,7 +268,7 @@ def handler(q=False):
else: else:
misperrors["error"] = "No information retrieved from VulnDB." misperrors["error"] = "No information retrieved from VulnDB."
return misperrors return misperrors
except: except Exception:
misperrors["error"] = "Error while fetching information from VulnDB, wrong API keys?" misperrors["error"] = "Error while fetching information from VulnDB, wrong API keys?"
return misperrors return misperrors

1
misp_modules/modules/expansion/vulners.py
View File

@ -1,5 +1,4 @@
import json import json
import requests
import vulners import vulners
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}

4
misp_modules/modules/expansion/wiki.py
View File

@ -1,5 +1,4 @@
import json import json
import requests
from SPARQLWrapper import SPARQLWrapper, JSON from SPARQLWrapper import SPARQLWrapper, JSON
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
@ -23,7 +22,7 @@ def handler(q=False):
"SELECT ?item \n" \ "SELECT ?item \n" \
"WHERE { \n" \ "WHERE { \n" \
"?item rdfs:label\"" + request.get('text') + "\" @en \n" \ "?item rdfs:label\"" + request.get('text') + "\" @en \n" \
"}\n"; "}\n"
sparql.setQuery(query_string) sparql.setQuery(query_string)
sparql.setReturnFormat(JSON) sparql.setReturnFormat(JSON)
results = sparql.query().convert() results = sparql.query().convert()
@ -47,4 +46,3 @@ def introspection():
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

8
misp_modules/modules/expansion/xforceexchange.py
View File

@ -27,7 +27,6 @@ moduleconfig = ["apikey", "event_limit"]
limit = 5000 # Default limit = 5000 # Default
def MyHeader(key=False): def MyHeader(key=False):
global limit global limit
if key is False: if key is False:
@ -37,6 +36,7 @@ def MyHeader(key=False):
"Accept": "application/json", "Accept": "application/json",
'User-Agent': 'Mozilla 5.0'} 'User-Agent': 'Mozilla 5.0'}
def handler(q=False): def handler(q=False):
global limit global limit
if q is False: if q is False:
@ -71,15 +71,16 @@ def handler(q=False):
r["results"] = uniq r["results"] = uniq
return r return r
def apicall(indicator_type, indicator, key=False): def apicall(indicator_type, indicator, key=False):
try: try:
myURL = BASEurl + (extensions[str(indicator_type)]) % indicator myURL = BASEurl + (extensions[str(indicator_type)]) % indicator
jsondata = requests.get(myURL, headers=MyHeader(key)).json() jsondata = requests.get(myURL, headers=MyHeader(key)).json()
except: except Exception:
jsondata = None jsondata = None
redata = [] redata = []
# print(jsondata) # print(jsondata)
if not jsondata is None: if jsondata is not None:
if indicator_type is "hash": if indicator_type is "hash":
if "malware" in jsondata: if "malware" in jsondata:
lopointer = jsondata["malware"] lopointer = jsondata["malware"]
@ -92,6 +93,7 @@ def apicall(indicator_type, indicator, key=False):
return redata return redata
def introspection(): def introspection():
return mispattributes return mispattributes

4
misp_modules/modules/expansion/yara_query.py
View File

@ -13,11 +13,13 @@ moduleinfo = {'version': '1', 'author': 'Christian STUDER',
moduleconfig = [] moduleconfig = []
mispattributes = {'input': ['md5', 'sha1', 'sha256', 'filename|md5', 'filename|sha1', 'filename|sha256', 'imphash'], 'output': ['yara']} mispattributes = {'input': ['md5', 'sha1', 'sha256', 'filename|md5', 'filename|sha1', 'filename|sha256', 'imphash'], 'output': ['yara']}
def get_hash_condition(hashtype, hashvalue): def get_hash_condition(hashtype, hashvalue):
hashvalue = hashvalue.lower() hashvalue = hashvalue.lower()
required_module, params = ('pe', '()') if hashtype == 'imphash' else ('hash', '(0, filesize)') required_module, params = ('pe', '()') if hashtype == 'imphash' else ('hash', '(0, filesize)')
return '{}.{}{} == "{}"'.format(required_module, hashtype, params, hashvalue), required_module return '{}.{}{} == "{}"'.format(required_module, hashtype, params, hashvalue), required_module
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -42,9 +44,11 @@ def handler(q=False):
return misperrors return misperrors
return {'results': [{'types': mispattributes['output'], 'values': rule}]} return {'results': [{'types': mispattributes['output'], 'values': rule}]}
def introspection(): def introspection():
return mispattributes return mispattributes
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

3
misp_modules/modules/expansion/yara_syntax_validator.py
View File

@ -1,5 +1,4 @@
import json import json
import requests
try: try:
import yara import yara
except (OSError, ImportError): except (OSError, ImportError):
@ -20,7 +19,7 @@ def handler(q=False):
return misperrors return misperrors
try: try:
rules = yara.compile(source=request.get('yara')) yara.compile(source=request.get('yara'))
summary = ("Syntax valid") summary = ("Syntax valid")
except Exception as e: except Exception as e:
summary = ("Syntax error: " + str(e)) summary = ("Syntax error: " + str(e))

3
misp_modules/modules/export_mod/__init__.py
View File

@ -1 +1,2 @@
__all__ = ['cef_export','liteexport','goamlexport','threat_connect_export','pdfexport','threatStream_misp_export', 'osqueryexport'] __all__ = ['cef_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport',
'threatStream_misp_export', 'osqueryexport']

6
misp_modules/modules/export_mod/cef_export.py
View File

@ -20,6 +20,7 @@ mispattributes = {'input':list(cefmapping.keys())}
outputFileExtension = "cef" outputFileExtension = "cef"
responseType = "application/txt" responseType = "application/txt"
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -27,7 +28,8 @@ def handler(q=False):
if "config" in request: if "config" in request:
config = request["config"] config = request["config"]
else: else:
config = {"Default_Severity":1, "Device_Vendor":"MISP", "Device_Product":"MISP", "Device_Version":1} config = {"Default_Severity": 1, "Device_Vendor": "MISP",
"Device_Product": "MISP", "Device_Version": 1}
data = request["data"] data = request["data"]
response = "" response = ""
@ -75,7 +77,7 @@ def introspection():
pass pass
return modulesetup return modulesetup
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

4
misp_modules/modules/export_mod/goamlexport.py
View File

@ -48,6 +48,7 @@ referencesMapping = {'bank-account': {'aml_type': '{}_account', 'bracket': 't_{}
'legal-entity': {'transaction': {'aml_type': '{}_entity', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_entity'}}, 'legal-entity': {'transaction': {'aml_type': '{}_entity', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_entity'}},
'geolocation': {'aml_type': 'address', 'bracket': 'addresses'}} 'geolocation': {'aml_type': 'address', 'bracket': 'addresses'}}
class GoAmlGeneration(object): class GoAmlGeneration(object):
def __init__(self, config): def __init__(self, config):
self.config = config self.config = config
@ -186,6 +187,7 @@ class GoAmlGeneration(object):
self.itterate(next_object_type, next_aml_type, uuid, xml_part) self.itterate(next_object_type, next_aml_type, uuid, xml_part)
self.xml[xml_part] += "</{}>".format(bracket) self.xml[xml_part] += "</{}>".format(bracket)
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -212,6 +214,7 @@ def handler(q=False):
exp_doc = "{}{}".format(export_doc.xml.get('header'), export_doc.xml.get('data')) exp_doc = "{}{}".format(export_doc.xml.get('header'), export_doc.xml.get('data'))
return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc, 'utf-8')), 'utf-8')} return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc, 'utf-8')), 'utf-8')}
def introspection(): def introspection():
modulesetup = {} modulesetup = {}
try: try:
@ -236,6 +239,7 @@ def introspection():
pass pass
return modulesetup return modulesetup
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

13
misp_modules/modules/export_mod/liteexport.py
View File

@ -14,6 +14,7 @@ mispattributes = {}
outputFileExtension = "json" outputFileExtension = "json"
responseType = "application/json" responseType = "application/json"
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -29,7 +30,7 @@ def handler(q=False):
if config['indent_json_export'] is not None: if config['indent_json_export'] is not None:
try: try:
config['indent_json_export'] = int(config['indent_json_export']) config['indent_json_export'] = int(config['indent_json_export'])
except: except Exception:
config['indent_json_export'] = None config['indent_json_export'] = None
if 'data' not in request: if 'data' not in request:
@ -53,12 +54,9 @@ def handler(q=False):
liteEvent['Event']['Attribute'].append(liteAttr) liteEvent['Event']['Attribute'].append(liteAttr)
return {'response': [], return {'response': [],
'data' : str(base64.b64encode( 'data': str(base64.b64encode(bytes(
bytes( json.dumps(liteEvent, indent=config['indent_json_export']), 'utf-8')), 'utf-8')}
json.dumps(liteEvent, indent=config['indent_json_export']),
'utf-8')),
'utf-8')
}
def introspection(): def introspection():
modulesetup = {} modulesetup = {}
@ -84,6 +82,7 @@ def introspection():
pass pass
return modulesetup return modulesetup
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

10
misp_modules/modules/export_mod/osqueryexport.py
View File

@ -13,7 +13,7 @@ types_to_use = ['regkey', 'regkey|value', 'mutex', 'windows-service-displayname'
userConfig = { userConfig = {
}; }
moduleconfig = [] moduleconfig = []
inputSource = ['event'] inputSource = ['event']
@ -26,6 +26,7 @@ moduleinfo = {'version': '1.0', 'author': 'Julien Bachmann, Hacknowledge',
'description': 'OSQuery query export module', 'description': 'OSQuery query export module',
'module-type': ['export']} 'module-type': ['export']}
def handle_regkey(value): def handle_regkey(value):
rep = {'HKCU': 'HKEY_USERS\\%', 'HKLM': 'HKEY_LOCAL_MACHINE'} rep = {'HKCU': 'HKEY_USERS\\%', 'HKLM': 'HKEY_LOCAL_MACHINE'}
rep = dict((re.escape(k), v) for k, v in rep.items()) rep = dict((re.escape(k), v) for k, v in rep.items())
@ -33,6 +34,7 @@ def handle_regkey(value):
value = pattern.sub(lambda m: rep[re.escape(m.group(0))], value) value = pattern.sub(lambda m: rep[re.escape(m.group(0))], value)
return 'SELECT * FROM registry WHERE path LIKE \'%s\';' % value return 'SELECT * FROM registry WHERE path LIKE \'%s\';' % value
def handle_regkeyvalue(value): def handle_regkeyvalue(value):
key, value = value.split('|') key, value = value.split('|')
rep = {'HKCU': 'HKEY_USERS\\%', 'HKLM': 'HKEY_LOCAL_MACHINE'} rep = {'HKCU': 'HKEY_USERS\\%', 'HKLM': 'HKEY_LOCAL_MACHINE'}
@ -41,18 +43,23 @@ def handle_regkeyvalue(value):
key = pattern.sub(lambda m: rep[re.escape(m.group(0))], key) key = pattern.sub(lambda m: rep[re.escape(m.group(0))], key)
return 'SELECT * FROM registry WHERE path LIKE \'%s\' AND data LIKE \'%s\';' % (key, value) return 'SELECT * FROM registry WHERE path LIKE \'%s\' AND data LIKE \'%s\';' % (key, value)
def handle_mutex(value): def handle_mutex(value):
return 'SELECT * FROM winbaseobj WHERE object_name LIKE \'%s\';' % value return 'SELECT * FROM winbaseobj WHERE object_name LIKE \'%s\';' % value
def handle_service(value): def handle_service(value):
return 'SELECT * FROM services WHERE display_name LIKE \'%s\' OR name like \'%s\';' % (value, value) return 'SELECT * FROM services WHERE display_name LIKE \'%s\' OR name like \'%s\';' % (value, value)
def handle_yara(value): def handle_yara(value):
return 'not implemented yet, not sure it\'s easily feasible w/o dropping the sig on the hosts first' return 'not implemented yet, not sure it\'s easily feasible w/o dropping the sig on the hosts first'
def handle_scheduledtask(value): def handle_scheduledtask(value):
return 'SELECT * FROM scheduled_tasks WHERE name LIKE \'%s\';' % value return 'SELECT * FROM scheduled_tasks WHERE name LIKE \'%s\';' % value
handlers = { handlers = {
'regkey': handle_regkey, 'regkey': handle_regkey,
'regkey|value': handle_regkeyvalue, 'regkey|value': handle_regkeyvalue,
@ -62,6 +69,7 @@ handlers = {
'yara': handle_yara 'yara': handle_yara
} }
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False

7
misp_modules/modules/export_mod/testexport.py
View File

@ -1,13 +1,12 @@
import json import json
import base64 import base64
import csv
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
userConfig = { userConfig = {
}; }
moduleconfig = [] moduleconfig = []
@ -28,8 +27,8 @@ def handler(q=False):
if q is False: if q is False:
return False return False
r = {'results': []} r = {'results': []}
result = json.loads(q) result = json.loads(q) # noqa
output = ''; # Insert your magic here! output = '' # Insert your magic here!
r = {"data": base64.b64encode(output.encode('utf-8')).decode('utf-8')} r = {"data": base64.b64encode(output.encode('utf-8')).decode('utf-8')}
return r return r

2
misp_modules/modules/export_mod/threatStream_misp_export.py
View File

@ -49,10 +49,8 @@ def handler(q=False):
if q is False or not q: if q is False or not q:
return False return False
request = json.loads(q) request = json.loads(q)
response = io.StringIO() response = io.StringIO()
writer = csv.DictWriter(response, fieldnames=["value", "itype", "tags"]) writer = csv.DictWriter(response, fieldnames=["value", "itype", "tags"])
writer.writeheader() writer.writeheader()

2
misp_modules/modules/import_mod/__init__.py
View File

@ -1,3 +1,3 @@
from . import _vmray from . import _vmray # noqa
__all__ = ['vmray_import', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport'] __all__ = ['vmray_import', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport']

22
misp_modules/modules/import_mod/csvimport.py
View File

@ -1,5 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import json, os, base64 import json
import os
import base64
import pymisp import pymisp
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
@ -21,6 +23,7 @@ duplicatedFields = {'mispType': {'mispComment': 'comment'},
attributesFields = ['type', 'value', 'category', 'to_ids', 'comment', 'distribution'] attributesFields = ['type', 'value', 'category', 'to_ids', 'comment', 'distribution']
delimiters = [',', ';', '|', '/', '\t', ' '] delimiters = [',', ';', '|', '/', '\t', ' ']
class CsvParser(): class CsvParser():
def __init__(self, header, has_header): def __init__(self, header, has_header):
self.header = header self.header = header
@ -32,17 +35,17 @@ class CsvParser():
return_data = [] return_data = []
if self.fields_number == 1: if self.fields_number == 1:
for line in data: for line in data:
l = line.split('#')[0].strip() line = line.split('#')[0].strip()
if l: if line:
return_data.append(l) return_data.append(line)
self.delimiter = None self.delimiter = None
else: else:
self.delimiter_count = dict([(d, 0) for d in delimiters]) self.delimiter_count = dict([(d, 0) for d in delimiters])
for line in data: for line in data:
l = line.split('#')[0].strip() line = line.split('#')[0].strip()
if l: if line:
self.parse_delimiter(l) self.parse_delimiter(line)
return_data.append(l) return_data.append(line)
# find which delimiter is used # find which delimiter is used
self.delimiter = self.find_delimiter() self.delimiter = self.find_delimiter()
self.data = return_data[1:] if self.has_header else return_data self.data = return_data[1:] if self.has_header else return_data
@ -115,6 +118,7 @@ class CsvParser():
# return list of indexes of the misp types, list of the misp types, remaining fields that will be attribute fields # return list of indexes of the misp types, list of the misp types, remaining fields that will be attribute fields
return list2pop, misp, list(reversed(head)) return list2pop, misp, list(reversed(head))
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -138,6 +142,7 @@ def handler(q=False):
r = {'results': csv_parser.attributes} r = {'results': csv_parser.attributes}
return r return r
def introspection(): def introspection():
modulesetup = {} modulesetup = {}
try: try:
@ -152,6 +157,7 @@ def introspection():
pass pass
return modulesetup return modulesetup
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

6
misp_modules/modules/import_mod/cuckooimport.py
View File

@ -1,6 +1,4 @@
import json import json
import logging
import sys
import base64 import base64
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
@ -13,6 +11,7 @@ moduleinfo = {'version': '0.1', 'author': 'Victor van der Stoep',
moduleconfig = [] moduleconfig = []
def handler(q=False): def handler(q=False):
# Just in case we have no data # Just in case we have no data
if q is False: if q is False:
@ -63,6 +62,7 @@ def handler(q=False):
# Return # Return
return r return r
def processSummary(r, summary): def processSummary(r, summary):
r["results"].append({ r["results"].append({
"values": summary['mutexes'], "values": summary['mutexes'],
@ -71,6 +71,7 @@ def processSummary(r, summary):
"comment": "Cuckoo analysis: Observed mutexes" "comment": "Cuckoo analysis: Observed mutexes"
}) })
def processVT(r, virustotal): def processVT(r, virustotal):
category = "Antivirus detection" category = "Antivirus detection"
comment = "VirusTotal analysis" comment = "VirusTotal analysis"
@ -187,6 +188,7 @@ def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo
if __name__ == '__main__': if __name__ == '__main__':
x = open('test.json', 'r') x = open('test.json', 'r')
q = [] q = []

7
misp_modules/modules/import_mod/email_import.py
View File

@ -115,7 +115,7 @@ def handler(q=False):
email_targets = set() email_targets = set()
for rec in received: for rec in received:
try: try:
email_check = re.search("for\s(.*@.*);", rec).group(1) email_check = re.search(r"for\s(.*@.*);", rec).group(1)
email_check = email_check.strip(' <>') email_check = email_check.strip(' <>')
email_targets.add(parseaddr(email_check)[1]) email_targets.add(parseaddr(email_check)[1])
except (AttributeError): except (AttributeError):
@ -166,7 +166,7 @@ def handler(q=False):
for ext in zipped_files: for ext in zipped_files:
if filename.endswith(ext) is True: if filename.endswith(ext) is True:
zipped_filetype = True zipped_filetype = True
if zipped_filetype == False: if not zipped_filetype:
try: try:
attachment_files += get_zipped_contents(filename, attachment_data) attachment_files += get_zipped_contents(filename, attachment_data)
except RuntimeError: # File is encrypted with a password except RuntimeError: # File is encrypted with a password
@ -294,7 +294,7 @@ def get_zip_passwords(message):
# Grab any strings that are marked off by special chars # Grab any strings that are marked off by special chars
marking_chars = [["\'", "\'"], ['"', '"'], ['[', ']'], ['(', ')']] marking_chars = [["\'", "\'"], ['"', '"'], ['[', ']'], ['(', ')']]
for char_set in marking_chars: for char_set in marking_chars:
regex = re.compile("""\{0}([^\{1}]*)\{1}""".format(char_set[0], char_set[1])) regex = re.compile(r"""\{0}([^\{1}]*)\{1}""".format(char_set[0], char_set[1]))
marked_off = re.findall(regex, raw_text) marked_off = re.findall(regex, raw_text)
possible_passwords += marked_off possible_passwords += marked_off
@ -397,6 +397,7 @@ def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo
if __name__ == '__main__': if __name__ == '__main__':
with open('tests/test_no_attach.eml', 'r') as email_file: with open('tests/test_no_attach.eml', 'r') as email_file:
handler(q=email_file.read()) handler(q=email_file.read())

11
misp_modules/modules/import_mod/goamlimport.py
View File

@ -1,6 +1,7 @@
import json, datetime, time, base64 import json
import time
import base64
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from collections import defaultdict
from pymisp import MISPEvent, MISPObject from pymisp import MISPEvent, MISPObject
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
@ -73,6 +74,7 @@ goAMLmapping = {'from_account': t_account_mapping, 'to_account': t_account_mappi
nodes_to_ignore = ['addresses', 'signatory'] nodes_to_ignore = ['addresses', 'signatory']
relationship_to_keep = ['signatory', 't_from', 't_from_my_client', 't_to', 't_to_my_client', 'address'] relationship_to_keep = ['signatory', 't_from', 't_from_my_client', 't_to', 't_to_my_client', 'address']
class GoAmlParser(): class GoAmlParser():
def __init__(self): def __init__(self):
self.misp_event = MISPEvent() self.misp_event = MISPEvent()
@ -145,6 +147,7 @@ class GoAmlParser():
to_country_attribute = {'object_relation': 'to-country', 'value': to_country} to_country_attribute = {'object_relation': 'to-country', 'value': to_country}
misp_object.add_attribute(**to_country_attribute) misp_object.add_attribute(**to_country_attribute)
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -157,16 +160,18 @@ def handler(q=False):
aml_parser = GoAmlParser() aml_parser = GoAmlParser()
try: try:
aml_parser.read_xml(data) aml_parser.read_xml(data)
except: except Exception:
misperrors['error'] = "Impossible to read XML data" misperrors['error'] = "Impossible to read XML data"
return misperrors return misperrors
aml_parser.parse_xml() aml_parser.parse_xml()
r = {'results': [obj.to_json() for obj in aml_parser.misp_event.objects]} r = {'results': [obj.to_json() for obj in aml_parser.misp_event.objects]}
return r return r
def introspection(): def introspection():
return mispattributes return mispattributes
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo

6
misp_modules/modules/import_mod/mispjson.py
View File

@ -2,7 +2,7 @@ import json
import base64 import base64
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
userConfig = { }; userConfig = {}
inputSource = ['file'] inputSource = ['file']
@ -32,10 +32,11 @@ def handler(q=False):
if a.get("data"): if a.get("data"):
tmp["data"] = a["data"] tmp["data"] = a["data"]
r['results'].append(tmp) r['results'].append(tmp)
except: except Exception:
pass pass
return r return r
def introspection(): def introspection():
modulesetup = {} modulesetup = {}
try: try:
@ -55,6 +56,7 @@ def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo
if __name__ == '__main__': if __name__ == '__main__':
x = open('test.json', 'r') x = open('test.json', 'r')
r = handler(q=x.read()) r = handler(q=x.read())

4
misp_modules/modules/import_mod/ocr.py
View File

@ -14,7 +14,7 @@ ch.setFormatter(formatter)
log.addHandler(ch) log.addHandler(ch)
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
userConfig = {}; userConfig = {}
inputSource = ['file'] inputSource = ['file']
@ -78,7 +78,6 @@ def handler(q=False):
misperrors['error'] = "Corrupt or not an image file." misperrors['error'] = "Corrupt or not an image file."
return misperrors return misperrors
ocrized = image_to_string(im) ocrized = image_to_string(im)
freetext = {} freetext = {}
@ -107,6 +106,7 @@ def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo
if __name__ == '__main__': if __name__ == '__main__':
x = open('test.json', 'r') x = open('test.json', 'r')
handler(q=x.read()) handler(q=x.read())

5
misp_modules/modules/import_mod/testimport.py
View File

@ -1,6 +1,5 @@
import json import json
import base64 import base64
import csv
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
userConfig = { userConfig = {
@ -22,7 +21,7 @@ userConfig = {
'type': 'Integer', 'type': 'Integer',
'message': 'Column number used for comment' 'message': 'Column number used for comment'
} }
}; }
inputSource = ['file', 'paste'] inputSource = ['file', 'paste']
@ -39,7 +38,7 @@ def handler(q=False):
r = {'results': []} r = {'results': []}
request = json.loads(q) request = json.loads(q)
request["data"] = base64.b64decode(request["data"]) request["data"] = base64.b64decode(request["data"])
fields = ["value", "category", "type", "comment"] # fields = ["value", "category", "type", "comment"]
r = {"results": [{"values": ["192.168.56.1"], "types":["ip-src"], "categories": ["Network activity"]}]} r = {"results": [{"values": ["192.168.56.1"], "types":["ip-src"], "categories": ["Network activity"]}]}
return r return r

4
misp_modules/modules/import_mod/threatanalyzer_import.py
View File

@ -90,7 +90,7 @@ def handler(q=False):
'values': sample_filename, 'values': sample_filename,
'data': base64.b64encode(file_data).decode(), 'data': base64.b64encode(file_data).decode(),
'type': 'malware-sample', 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': True, 'comment': ''}) 'type': 'malware-sample', 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': True, 'comment': ''})
except Exception as e: except Exception:
# no 'sample' in archive, might be an url analysis, just ignore # no 'sample' in archive, might be an url analysis, just ignore
pass pass
@ -171,7 +171,7 @@ def process_analysis_json(analysis_json):
if ':' in val: if ':' in val:
try: try:
val_port = int(val.split(':')[1]) val_port = int(val.split(':')[1])
except ValueError as e: except ValueError:
val_port = False val_port = False
val_hostname = cleanup_hostname(val.split(':')[0]) val_hostname = cleanup_hostname(val.split(':')[0])
val_ip = cleanup_ip(val.split(':')[0]) val_ip = cleanup_ip(val.split(':')[0])

2
misp_modules/modules/import_mod/vmray_import.py
View File

@ -117,7 +117,7 @@ def handler(q=False):
else: else:
misperrors['error'] = "Unable to fetch sample id %u" % (sample_id) misperrors['error'] = "Unable to fetch sample id %u" % (sample_id)
return misperrors return misperrors
except: except Exception:
misperrors['error'] = "Unable to access VMRay API" misperrors['error'] = "Unable to access VMRay API"
return misperrors return misperrors
else: else: