MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

fix: Make pep8 happy

pipenv
Raphaël Vinot 2018-12-11 15:29:09 +01:00
parent d0aec62f1a
commit 8fc5b1fd1f
56 changed files with 695 additions and 638 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -13,7 +13,7 @@ python:
- "3.7-dev"
install:
- pip install -U nose codecov pytest
- pip install -U nose codecov pytest flake8
- pip install -U -r REQUIREMENTS
- pip install .
@ -30,6 +30,7 @@ script:
- sleep 5
- nosetests --with-coverage --cover-package=misp_modules
- kill -s INT $pid
- flake8 --ignore=E501,W503 misp_modules
after_success:
- coverage combine .coverage*

9
misp_modules/__init__.py
View File

@ -38,14 +38,14 @@ from tornado.concurrent import run_on_executor
from concurrent.futures import ThreadPoolExecutor
try:
from .modules import *
from .modules import * # noqa
HAS_PACKAGE_MODULES = True
except Exception as e:
print(e)
HAS_PACKAGE_MODULES = False
try:
from .helpers import *
from .helpers import * # noqa
HAS_PACKAGE_HELPERS = True
except Exception as e:
print(e)
@ -148,7 +148,7 @@ def load_package_modules():
mhandlers = {}
modules = []
for path, module in sys.modules.items():
r = re.findall("misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
r = re.findall(r"misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
if r and len(r[0]) == 2:
moduletype, modulename = r[0]
mhandlers[modulename] = module
@ -159,6 +159,9 @@ def load_package_modules():
class ListModules(tornado.web.RequestHandler):
global loaded_modules
global mhandlers
def get(self):
ret = []
for module in loaded_modules:

3
misp_modules/helpers/cache.py
View File

@ -33,7 +33,7 @@ def selftest(enable=True):
r = redis.StrictRedis(host=hostname, port=port, db=db)
try:
r.ping()
except:
except Exception:
return 'Redis not running or not installed. Helper will be disabled.'
@ -62,6 +62,7 @@ def flush():
returncode = r.flushdb()
return returncode
if __name__ == "__main__":
import sys
if selftest() is not None:

6
misp_modules/modules/__init__.py
View File

@ -1,3 +1,3 @@
from .expansion import *
from .import_mod import *
from .export_mod import *
from .expansion import * # noqa
from .import_mod import * # noqa
from .export_mod import * # noqa

2
misp_modules/modules/expansion/__init__.py
View File

@ -1,4 +1,4 @@
from . import _vmray
from . import _vmray # noqa
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl',
'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi',

59
misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py
View File

@ -47,9 +47,11 @@ options = None
locale.setlocale(locale.LC_ALL, '')
class QueryError(Exception):
pass
class DnsdbClient(object):
def __init__(self, server, apikey, limit=None, http_proxy=None, https_proxy=None):
self.server = server
@ -81,7 +83,6 @@ class DnsdbClient(object):
return self._query(path, before, after)
def _query(self, path, before=None, after=None):
res = []
url = '%s/lookup/%s' % (self.server, path)
params = {}
@ -120,12 +121,15 @@ class DnsdbClient(object):
except (HTTPError, URLError) as e:
raise QueryError(str(e), sys.exc_traceback)
def quote(path):
return urllib_quote(path, safe='')
def sec_to_text(ts):
return time.strftime('%Y-%m-%d %H:%M:%S -0000', time.gmtime(ts))
def rrset_to_text(m):
s = StringIO()
@ -155,9 +159,11 @@ def rrset_to_text(m):
finally:
s.close()
def rdata_to_text(m):
return '%s IN %s %s' % (m['rrname'], m['rrtype'], m['rdata'])
def parse_config(cfg_files):
config = {}
@ -172,6 +178,7 @@ def parse_config(cfg_files):
return config
def time_parse(s):
try:
epoch = int(s)
@ -193,14 +200,15 @@ def time_parse(s):
m = re.match(r'^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$', s, re.I)
if m:
return -1*(int(m.group(1) or 0)*604800 +
int(m.group(2) or 0)*86400+
int(m.group(3) or 0)*3600+
int(m.group(4) or 0)*60+
int(m.group(5) or 0))
return -1 * (int(m.group(1) or 0) * 604800
+ int(m.group(2) or 0) * 86400
+ int(m.group(3) or 0) * 3600
+ int(m.group(4) or 0) * 60
+ int(m.group(5) or 0))
raise ValueError('Invalid time: "%s"' % s)
def epipe_wrapper(func):
def f(*args, **kwargs):
try:
@ -211,31 +219,23 @@ def epipe_wrapper(func):
raise
return f
@epipe_wrapper
def main():
global cfg
global options
parser = optparse.OptionParser(epilog='Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)')
parser.add_option('-c', '--config', dest='config',
help='config file', action='append')
parser.add_option('-r', '--rrset', dest='rrset', type='string',
help='rrset <ONAME>[/<RRTYPE>[/BAILIWICK]]')
parser.add_option('-n', '--rdataname', dest='rdata_name', type='string',
help='rdata name <NAME>[/<RRTYPE>]')
parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string',
help='rdata ip <IPADDRESS|IPRANGE|IPNETWORK>')
parser.add_option('-t', '--rrtype', dest='rrtype', type='string',
help='rrset or rdata rrtype')
parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string',
help='rrset bailiwick')
parser.add_option('-c', '--config', dest='config', help='config file', action='append')
parser.add_option('-r', '--rrset', dest='rrset', type='string', help='rrset <ONAME>[/<RRTYPE>[/BAILIWICK]]')
parser.add_option('-n', '--rdataname', dest='rdata_name', type='string', help='rdata name <NAME>[/<RRTYPE>]')
parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string', help='rdata ip <IPADDRESS|IPRANGE|IPNETWORK>')
parser.add_option('-t', '--rrtype', dest='rrtype', type='string', help='rrset or rdata rrtype')
parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string', help='rrset bailiwick')
parser.add_option('-s', '--sort', dest='sort', type='string', help='sort key')
parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False,
help='reverse sort')
parser.add_option('-j', '--json', dest='json', action='store_true', default=False,
help='output in JSON format')
parser.add_option('-l', '--limit', dest='limit', type='int', default=0,
help='limit number of results')
parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False, help='reverse sort')
parser.add_option('-j', '--json', dest='json', action='store_true', default=False, help='output in JSON format')
parser.add_option('-l', '--limit', dest='limit', type='int', default=0, help='limit number of results')
parser.add_option('', '--before', dest='before', type='string', help='only output results seen before this time')
parser.add_option('', '--after', dest='after', type='string', help='only output results seen after this time')
@ -263,13 +263,13 @@ def main():
print(str(e), file=sys.stderr)
sys.exit(1)
if not 'DNSDB_SERVER' in cfg:
if 'DNSDB_SERVER' not in cfg:
cfg['DNSDB_SERVER'] = DEFAULT_DNSDB_SERVER
if not 'HTTP_PROXY' in cfg:
if 'HTTP_PROXY' not in cfg:
cfg['HTTP_PROXY'] = DEFAULT_HTTP_PROXY
if not 'HTTPS_PROXY' in cfg:
if 'HTTPS_PROXY' not in cfg:
cfg['HTTPS_PROXY'] = DEFAULT_HTTPS_PROXY
if not 'APIKEY' in cfg:
if 'APIKEY' not in cfg:
sys.stderr.write('dnsdb_query: APIKEY not defined in config file\n')
sys.exit(1)
@ -307,7 +307,7 @@ def main():
if options.sort:
results = list(results)
if len(results) > 0:
if not options.sort in results[0]:
if options.sort not in results[0]:
sort_keys = results[0].keys()
sort_keys.sort()
sys.stderr.write('dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' % (options.sort, ', '.join(sort_keys)))
@ -319,5 +319,6 @@ def main():
print(e.message, file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
main()

6
misp_modules/modules/expansion/asn_history.py
View File

@ -22,9 +22,9 @@ def handler(q=False):
misperrors['error'] = "Unsupported attributes type"
return misperrors
if not request.get('config') and not (request['config'].get('host') and
request['config'].get('port') and
request['config'].get('db')):
if not request.get('config') and not (request['config'].get('host')
and request['config'].get('port')
and request['config'].get('db')):
misperrors['error'] = 'ASN description history configuration is missing'
return misperrors

23
misp_modules/modules/expansion/btc_steroids.py
View File

@ -15,21 +15,22 @@ moduleconfig = []
blockchain_firstseen = 'https://blockchain.info/q/addressfirstseen/'
blockchain_balance = 'https://blockchain.info/q/addressbalance/'
blockchain_totalreceived = 'https://blockchain.info/q/getreceivedbyaddress/'
blockchain_all='https://blockchain.info/rawaddr/'
converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts='
blockchain_all = 'https://blockchain.info/rawaddr/{}?filter=5{}'
converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts={}'
converter_rls = 'https://min-api.cryptocompare.com/stats/rate/limit'
result_text = ""
g_rate_limit = 300
start_time = 0
conversion_rates = {}
def get_consumption(output=False):
try:
req = requests.get(converter_rls)
jreq = req.json()
minute = str(jreq['Data']['calls_left']['minute'])
hour = str(jreq['Data']['calls_left']['hour'])
except:
except Exception:
minute = str(-1)
hour = str(-1)
# Debug out for the console
@ -53,7 +54,7 @@ def convert(btc, timestamp):
minute, hour = get_consumption()
g_rate_limit -= 1
now = time.time()
delta = now - start_time
# delta = now - start_time
# print(g_rate_limit)
if g_rate_limit <= 10:
minute, hour = get_consumption(output=True)
@ -66,7 +67,7 @@ def convert(btc, timestamp):
start_time = time.time()
g_rate_limit = int(minute)
try:
req = requests.get(converter+str(timestamp))
req = requests.get(converter.format(timestamp))
jreq = req.json()
usd = jreq['BTC']['USD']
eur = jreq['BTC']['EUR']
@ -90,8 +91,8 @@ def mprint(input):
def handler(q=False):
global result_text
global conversion_rates
start_time = time.time()
now = time.time()
# start_time = time.time()
# now = time.time()
if q is False:
return False
request = json.loads(q)
@ -107,9 +108,9 @@ def handler(q=False):
mprint("\nAddress:\t" + btc)
try:
req = requests.get(blockchain_all+btc+"?limit=50&filter=5")
req = requests.get(blockchain_all.format(btc, "&limit=50"))
jreq = req.json()
except Exception as e:
except Exception:
# print(e)
print(req.text)
result_text = ""
@ -130,11 +131,11 @@ def handler(q=False):
i = 0
while i < n_tx:
if click is False:
req = requests.get(blockchain_all+btc+"?limit=5&offset="+str(i)+"&filter=5")
req = requests.get(blockchain_all.format(btc, "&limit=5&offset={}".format(i)))
if n_tx > 5:
n_tx = 5
else:
req = requests.get(blockchain_all+btc+"?limit=50&offset="+str(i)+"&filter=5")
req = requests.get(blockchain_all.format(btc, "&limit=50&offset={}".format(i)))
jreq = req.json()
if jreq['txs']:
for transactions in jreq['txs']:

2
misp_modules/modules/expansion/countrycode.py
View File

@ -22,6 +22,7 @@ common_tlds = {"com":"Commercial (Worldwide)",
codes = False
def handler(q=False):
global codes
if not codes:
@ -58,4 +59,3 @@ def introspection():
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

7
misp_modules/modules/expansion/dbl_spamhaus.py
View File

@ -1,6 +1,5 @@
import json
import datetime
from collections import defaultdict
import sys
try:
import dns.resolver
@ -30,12 +29,14 @@ dbl_mapping = {'127.0.1.2': 'spam domain',
'127.0.1.106': 'abused legit botnet C&C',
'127.0.1.255': 'IP queries prohibited!'}
def fetch_requested_value(request):
for attribute_type in mispattributes['input']:
if request.get(attribute_type):
return request[attribute_type].split('|')[0]
return None
def handler(q=False):
if q is False:
return False
@ -52,9 +53,11 @@ def handler(q=False):
result = str(e)
return {'results': [{'types': mispattributes.get('output'), 'values': result}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

2
misp_modules/modules/expansion/dns.py
View File

@ -43,7 +43,7 @@ def handler(q=False):
except dns.exception.Timeout:
misperrors['error'] = "Timeout"
return misperrors
except:
except Exception:
misperrors['error'] = "DNS resolving error"
return misperrors

6
misp_modules/modules/expansion/farsight_passivedns.py
View File

@ -51,7 +51,7 @@ def lookup_name(client, name):
for i in item.get('rdata'):
# grab email field and replace first dot by @ to convert to an email address
yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1))
except QueryError as e:
except QueryError:
pass
try:
@ -59,7 +59,7 @@ def lookup_name(client, name):
for item in res:
if item.get('rrtype') in ['A', 'AAAA', 'CNAME']:
yield(item.get('rrname').rstrip('.'))
except QueryError as e:
except QueryError:
pass
@ -68,7 +68,7 @@ def lookup_ip(client, ip):
res = client.query_rdata_ip(ip)
for item in res:
yield(item['rrname'].rstrip('.'))
except QueryError as e:
except QueryError:
pass

4
misp_modules/modules/expansion/geoip_country.py
View File

@ -27,7 +27,7 @@ try:
config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'geoip_country.cfg'))
gi = pygeoip.GeoIP(config.get('GEOIP', 'database'))
enabled = True
except:
except Exception:
enabled = False
@ -49,7 +49,7 @@ def handler(q=False):
try:
answer = gi.country_code_by_addr(toquery)
except:
except Exception:
misperrors['error'] = "GeoIP resolving error"
return misperrors

6
misp_modules/modules/expansion/ipasn.py
View File

@ -24,9 +24,9 @@ def handler(q=False):
misperrors['error'] = "Unsupported attributes type"
return misperrors
if not request.get('config') and not (request['config'].get('host') and
request['config'].get('port') and
request['config'].get('db')):
if not request.get('config') and not (request['config'].get('host')
and request['config'].get('port')
and request['config'].get('db')):
misperrors['error'] = 'IP ASN history configuration is missing'
return misperrors

4
misp_modules/modules/expansion/iprep.py
View File

@ -45,7 +45,7 @@ def parse_iprep(ip, api):
url = 'https://www.packetmail.net/iprep.php/%s' % ip
try:
data = requests.get(url, params={'apikey': api}).json()
except:
except Exception:
return ['Error pulling data'], rep
# print '%s' % data
for name, val in data.items():
@ -72,7 +72,7 @@ def parse_iprep(ip, api):
full_text += '\n%s' % context
misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen'])
rep.append({'types': mispattributes['output'], 'categories': ['External analysis'], 'values': misp_val, 'comment': misp_comment})
except:
except Exception:
err.append('Error parsing source: %s' % name)
rep.append({'types': ['freetext'], 'values': full_text, 'comment': 'Free text import of IPRep'})

8
misp_modules/modules/expansion/macaddress_io.py
View File

@ -86,8 +86,8 @@ def handler(q=False):
response.block_details.date_updated.strftime('%d %B %Y') if response.block_details.date_updated else None
results = {
'results': [
{'types': ['text'], 'values':
'results':
[{'types': ['text'], 'values':
{
# Mac address details
'Valid MAC address': "True" if response.mac_address_details.is_valid else "False",
@ -109,9 +109,7 @@ def handler(q=False):
'Assignment block size': response.block_details.assignment_block_size,
'Date when the range was allocated': date_created,
'Date when the range was last updated': date_updated
}
}
]
}}]
}
return results

6
misp_modules/modules/expansion/onyphe.py
View File

@ -1,4 +1,3 @@
import json
# -*- coding: utf-8 -*-
import json
@ -9,7 +8,8 @@ except ImportError:
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], 'output': ['hostname', 'domain', 'ip-src', 'ip-dst','url']}
mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']}
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
'description': 'Query on Onyphe',
@ -54,7 +54,7 @@ def handle_expansion(api, ip, misperrors):
misperrors['error'] = result['message']
return misperrors
categories = list(set([item['@category'] for item in result['results']]))
# categories = list(set([item['@category'] for item in result['results']]))
result_filtered = {"results": []}
urls_pasties = []

4
misp_modules/modules/expansion/onyphe_full.py
View File

@ -1,4 +1,3 @@
import json
# -*- coding: utf-8 -*-
import json
@ -188,7 +187,7 @@ def expand_syscan(api, ip, misperror):
def expand_datascan(api, misperror, **kwargs):
status_ok = False
r = []
ip = ''
# ip = ''
query = ''
asn_list = []
geoloc = []
@ -368,6 +367,7 @@ def expand_threatlist(api, misperror,**kwargs):
return r, status_ok
def introspection():
return mispattributes

8
misp_modules/modules/expansion/otx.py
View File

@ -15,6 +15,7 @@ moduleinfo = {'version': '1', 'author': 'chrisdoman',
# We're not actually using the API key yet
moduleconfig = ["apikey"]
# Avoid adding windows update to enrichment etc.
def isBlacklisted(value):
blacklist = ['0.0.0.0', '8.8.8.8', '255.255.255.255', '192.168.56.', 'time.windows.com']
@ -25,10 +26,12 @@ def isBlacklisted(value):
return True
def valid_ip(ip):
m = re.match(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$", ip)
return bool(m) and all(map(lambda n: 0 <= int(n) <= 255, m.groups()))
def findAll(data, keys):
a = []
if isinstance(data, dict):
@ -43,9 +46,11 @@ def findAll(data, keys):
a.extend(findAll(i, keys))
return a
def valid_email(email):
return bool(re.search(r"[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?", email))
def handler(q=False):
if q is False:
return False
@ -104,14 +109,12 @@ def getIP(ip, key):
for _hash in findAll(req, "hash"):
ret.append({"types": ["sha256"], "values": [_hash]})
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text)
for hostname in findAll(req, "hostname"):
if not isBlacklisted(hostname):
ret.append({"types": ["hostname"], "values": [hostname]})
return ret
@ -144,6 +147,7 @@ def getDomain(domain, key):
return ret
def introspection():
return mispattributes

2
misp_modules/modules/expansion/passivetotal.py
View File

@ -331,7 +331,7 @@ def handler(q=False):
output['results'] += results
else:
log.error("Unsupported query pattern issued.")
except:
except Exception:
return misperrors
return output

5
misp_modules/modules/expansion/rbl.py
View File

@ -1,5 +1,5 @@
import json
import datetime
import sys
try:
import dns.resolver
@ -76,6 +76,7 @@ rbls = {
'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html'
}
def handler(q=False):
if q is False:
return False
@ -101,9 +102,11 @@ def handler(q=False):
result = "\n".join(["{}: {}".format(l, " - ".join(i)) for l, i in zip(listed, info)])
return {'results': [{'types': mispattributes.get('output'), 'values': result}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

15
misp_modules/modules/expansion/reversedns.py
View File

@ -1,5 +1,5 @@
import json
import dns.reversename, dns.resolver
from dns import reversename, resolver, exception
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['hostname']}
@ -12,6 +12,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
# config fields that your code expects from the site admin
moduleconfig = ['nameserver']
def handler(q=False):
if q is False:
return False
@ -26,9 +27,9 @@ def handler(q=False):
return False
# reverse lookup for ip
revname = dns.reversename.from_address(toquery)
revname = reversename.from_address(toquery)
r = dns.resolver.Resolver()
r = resolver.Resolver()
r.timeout = 2
r.lifetime = 2
@ -42,13 +43,13 @@ def handler(q=False):
try:
answer = r.query(revname, 'PTR')
except dns.resolver.NXDOMAIN:
except resolver.NXDOMAIN:
misperrors['error'] = "NXDOMAIN"
return misperrors
except dns.exception.Timeout:
except exception.Timeout:
misperrors['error'] = "Timeout"
return misperrors
except:
except Exception:
misperrors['error'] = "DNS resolving error"
return misperrors
@ -56,9 +57,11 @@ def handler(q=False):
'values':[str(answer[0])]}]}
return r
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

6
misp_modules/modules/expansion/securitytrails.py
View File

@ -114,8 +114,7 @@ def handle_domain(api, domain, misperrors):
if r:
result_filtered['results'].extend(r)
else:
misperrors['error'] = misperrors[
'error'] + ' Error in expand History DNS'
misperrors['error'] = misperrors['error'] + ' Error in expand History DNS'
return misperrors
r, status_ok = expand_history_whois(api, domain)
@ -124,8 +123,7 @@ def handle_domain(api, domain, misperrors):
if r:
result_filtered['results'].extend(r)
else:
misperrors['error'] = misperrors['error'] + \
' Error in expand History Whois'
misperrors['error'] = misperrors['error'] + ' Error in expand History Whois'
return misperrors
return result_filtered

9
misp_modules/modules/expansion/sigma_queries.py
View File

@ -1,4 +1,6 @@
import sys, os, io, json
import sys
import io
import json
try:
from sigma.parser import SigmaCollectionParser
from sigma.config import SigmaConfiguration
@ -13,6 +15,7 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['e
moduleconfig = []
sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'wdatp', 'splunkxml', 'arcsight', 'qualys')
def handler(q=False):
if q is False:
return False
@ -35,16 +38,18 @@ def handler(q=False):
backend.finalize()
print("#NEXT")
targets.append(t)
except:
except Exception:
continue
sys.stdout = old_stdout
results = result.getvalue()[:-5].split('#NEXT')
d_result = {t: r.strip() for t, r in zip(targets, results)}
return {'results': [{'types': mispattributes['output'], 'values': d_result}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

3
misp_modules/modules/expansion/sigma_syntax_validator.py
View File

@ -12,6 +12,7 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['e
'description': 'An expansion hover module to perform a syntax check on sigma rules'}
moduleconfig = []
def handler(q=False):
if q is False:
return False
@ -27,9 +28,11 @@ def handler(q=False):
result = ("Syntax error: {}".format(str(e)))
return {'results': [{'types': mispattributes['output'], 'values': result}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

3
misp_modules/modules/expansion/stix2_pattern_syntax_validator.py
View File

@ -10,6 +10,7 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['e
'description': 'An expansion hover module to perform a syntax check on stix2 patterns.'}
moduleconfig = []
def handler(q=False):
if q is False:
return False
@ -34,9 +35,11 @@ def handler(q=False):
result = "Syntax valid"
return {'results': [{'types': mispattributes['output'], 'values': result}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

12
misp_modules/modules/expansion/threatcrowd.py
View File

@ -25,28 +25,31 @@ def isBlacklisted(value):
return False
def valid_ip(ip):
m = re.match(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$", ip)
return bool(m) and all(map(lambda n: 0 <= int(n) <= 255, m.groups()))
def valid_domain(hostname):
if len(hostname) > 255:
return False
if hostname[-1] == ".":
hostname = hostname[:-1] # strip exactly one dot from the right, if present
allowed = re.compile("(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
allowed = re.compile(r"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
return all(allowed.match(x) for x in hostname.split("."))
def valid_email(email):
return bool(re.search(r"^[\w\.\+\-]+\@[\w]+\.[a-z]{2,3}$", email))
def handler(q=False):
if q is False:
return False
q = json.loads(q)
r = {"results": []}
if "ip-src" in q:
@ -110,11 +113,9 @@ def getIP(ip):
for hash in req["hashes"]:
ret.append({"types": ["md5"], "values": [hash]})
return ret
def getEmail(email):
ret = []
j = requests.get("https://www.threatcrowd.org/searchApi/v2/email/report/?email=" + email).text
@ -129,7 +130,6 @@ def getEmail(email):
return ret
def getDomain(domain):
ret = []
@ -148,9 +148,9 @@ def getDomain(domain):
for hash in req["hashes"]:
ret.append({"types": ["md5"], "values": [hash]})
return ret
def introspection():
return mispattributes

2
misp_modules/modules/expansion/threatminer.py
View File

@ -1,7 +1,5 @@
import json
import requests
from requests import HTTPError
import base64
misperrors = {'error': 'Error'}
mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'md5', 'sha1', 'sha256', 'sha512'],

4
misp_modules/modules/expansion/virustotal.py
View File

@ -19,6 +19,7 @@ moduleconfig = ["apikey", "event_limit"]
comment = '{}: Enriched via VirusTotal'
hash_types = ["md5", "sha1", "sha256", "sha512"]
class VirusTotalRequest(object):
def __init__(self, config):
self.apikey = config['apikey']
@ -146,6 +147,7 @@ class VirusTotalRequest(object):
self.to_return.append({"types": ["malware-sample"], "categories": ["Payload delivery"],
"values": data["submimssion_names"], "data": str(base64.b64encore(malsample), 'utf-8')})
def handler(q=False):
if q is False:
return False
@ -161,9 +163,11 @@ def handler(q=False):
return misperrors
return {'results': r}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

6
misp_modules/modules/expansion/vmray_submit.py
View File

@ -55,7 +55,7 @@ def handler(q=False):
else:
misperrors['error'] = "No malware sample or attachment supplied"
return misperrors
except:
except Exception:
misperrors['error'] = "Unable to process submited sample data"
return misperrors
@ -102,7 +102,7 @@ def handler(q=False):
return misperrors
else:
return vmrayProcess(vmraydata)
except:
except Exception:
misperrors['error'] = "Problem when calling API."
return misperrors
else:
@ -148,7 +148,7 @@ def vmrayProcess(vmraydata):
else:
misperrors['error'] = "No valid results returned."
return misperrors
except:
except Exception:
misperrors['error'] = "No valid submission data returned."
return misperrors
else:

2
misp_modules/modules/expansion/vulndb.py
View File

@ -268,7 +268,7 @@ def handler(q=False):
else:
misperrors["error"] = "No information retrieved from VulnDB."
return misperrors
except:
except Exception:
misperrors["error"] = "Error while fetching information from VulnDB, wrong API keys?"
return misperrors

1
misp_modules/modules/expansion/vulners.py
View File

@ -1,5 +1,4 @@
import json
import requests
import vulners
misperrors = {'error': 'Error'}

4
misp_modules/modules/expansion/wiki.py
View File

@ -1,5 +1,4 @@
import json
import requests
from SPARQLWrapper import SPARQLWrapper, JSON
misperrors = {'error': 'Error'}
@ -23,7 +22,7 @@ def handler(q=False):
"SELECT ?item \n" \
"WHERE { \n" \
"?item rdfs:label\"" + request.get('text') + "\" @en \n" \
"}\n";
"}\n"
sparql.setQuery(query_string)
sparql.setReturnFormat(JSON)
results = sparql.query().convert()
@ -47,4 +46,3 @@ def introspection():
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

8
misp_modules/modules/expansion/xforceexchange.py
View File

@ -27,7 +27,6 @@ moduleconfig = ["apikey", "event_limit"]
limit = 5000 # Default
def MyHeader(key=False):
global limit
if key is False:
@ -37,6 +36,7 @@ def MyHeader(key=False):
"Accept": "application/json",
'User-Agent': 'Mozilla 5.0'}
def handler(q=False):
global limit
if q is False:
@ -71,15 +71,16 @@ def handler(q=False):
r["results"] = uniq
return r
def apicall(indicator_type, indicator, key=False):
try:
myURL = BASEurl + (extensions[str(indicator_type)]) % indicator
jsondata = requests.get(myURL, headers=MyHeader(key)).json()
except:
except Exception:
jsondata = None
redata = []
# print(jsondata)
if not jsondata is None:
if jsondata is not None:
if indicator_type is "hash":
if "malware" in jsondata:
lopointer = jsondata["malware"]
@ -92,6 +93,7 @@ def apicall(indicator_type, indicator, key=False):
return redata
def introspection():
return mispattributes

4
misp_modules/modules/expansion/yara_query.py
View File

@ -13,11 +13,13 @@ moduleinfo = {'version': '1', 'author': 'Christian STUDER',
moduleconfig = []
mispattributes = {'input': ['md5', 'sha1', 'sha256', 'filename|md5', 'filename|sha1', 'filename|sha256', 'imphash'], 'output': ['yara']}
def get_hash_condition(hashtype, hashvalue):
hashvalue = hashvalue.lower()
required_module, params = ('pe', '()') if hashtype == 'imphash' else ('hash', '(0, filesize)')
return '{}.{}{} == "{}"'.format(required_module, hashtype, params, hashvalue), required_module
def handler(q=False):
if q is False:
return False
@ -42,9 +44,11 @@ def handler(q=False):
return misperrors
return {'results': [{'types': mispattributes['output'], 'values': rule}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

3
misp_modules/modules/expansion/yara_syntax_validator.py
View File

@ -1,5 +1,4 @@
import json
import requests
try:
import yara
except (OSError, ImportError):
@ -20,7 +19,7 @@ def handler(q=False):
return misperrors
try:
rules = yara.compile(source=request.get('yara'))
yara.compile(source=request.get('yara'))
summary = ("Syntax valid")
except Exception as e:
summary = ("Syntax error: " + str(e))

3
misp_modules/modules/export_mod/__init__.py
View File

@ -1 +1,2 @@
__all__ = ['cef_export','liteexport','goamlexport','threat_connect_export','pdfexport','threatStream_misp_export', 'osqueryexport']
__all__ = ['cef_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport',
'threatStream_misp_export', 'osqueryexport']

6
misp_modules/modules/export_mod/cef_export.py
View File

@ -20,6 +20,7 @@ mispattributes = {'input':list(cefmapping.keys())}
outputFileExtension = "cef"
responseType = "application/txt"
def handler(q=False):
if q is False:
return False
@ -27,7 +28,8 @@ def handler(q=False):
if "config" in request:
config = request["config"]
else:
config = {"Default_Severity":1, "Device_Vendor":"MISP", "Device_Product":"MISP", "Device_Version":1}
config = {"Default_Severity": 1, "Device_Vendor": "MISP",
"Device_Product": "MISP", "Device_Version": 1}
data = request["data"]
response = ""
@ -75,7 +77,7 @@ def introspection():
pass
return modulesetup
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

4
misp_modules/modules/export_mod/goamlexport.py
View File

@ -48,6 +48,7 @@ referencesMapping = {'bank-account': {'aml_type': '{}_account', 'bracket': 't_{}
'legal-entity': {'transaction': {'aml_type': '{}_entity', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_entity'}},
'geolocation': {'aml_type': 'address', 'bracket': 'addresses'}}
class GoAmlGeneration(object):
def __init__(self, config):
self.config = config
@ -186,6 +187,7 @@ class GoAmlGeneration(object):
self.itterate(next_object_type, next_aml_type, uuid, xml_part)
self.xml[xml_part] += "</{}>".format(bracket)
def handler(q=False):
if q is False:
return False
@ -212,6 +214,7 @@ def handler(q=False):
exp_doc = "{}{}".format(export_doc.xml.get('header'), export_doc.xml.get('data'))
return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc, 'utf-8')), 'utf-8')}
def introspection():
modulesetup = {}
try:
@ -236,6 +239,7 @@ def introspection():
pass
return modulesetup
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

13
misp_modules/modules/export_mod/liteexport.py
View File

@ -14,6 +14,7 @@ mispattributes = {}
outputFileExtension = "json"
responseType = "application/json"
def handler(q=False):
if q is False:
return False
@ -29,7 +30,7 @@ def handler(q=False):
if config['indent_json_export'] is not None:
try:
config['indent_json_export'] = int(config['indent_json_export'])
except:
except Exception:
config['indent_json_export'] = None
if 'data' not in request:
@ -53,12 +54,9 @@ def handler(q=False):
liteEvent['Event']['Attribute'].append(liteAttr)
return {'response': [],
'data' : str(base64.b64encode(
bytes(
json.dumps(liteEvent, indent=config['indent_json_export']),
'utf-8')),
'utf-8')
}
'data': str(base64.b64encode(bytes(
json.dumps(liteEvent, indent=config['indent_json_export']), 'utf-8')), 'utf-8')}
def introspection():
modulesetup = {}
@ -84,6 +82,7 @@ def introspection():
pass
return modulesetup
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

10
misp_modules/modules/export_mod/osqueryexport.py
View File

@ -13,7 +13,7 @@ types_to_use = ['regkey', 'regkey|value', 'mutex', 'windows-service-displayname'
userConfig = {
};
}
moduleconfig = []
inputSource = ['event']
@ -26,6 +26,7 @@ moduleinfo = {'version': '1.0', 'author': 'Julien Bachmann, Hacknowledge',
'description': 'OSQuery query export module',
'module-type': ['export']}
def handle_regkey(value):
rep = {'HKCU': 'HKEY_USERS\\%', 'HKLM': 'HKEY_LOCAL_MACHINE'}
rep = dict((re.escape(k), v) for k, v in rep.items())
@ -33,6 +34,7 @@ def handle_regkey(value):
value = pattern.sub(lambda m: rep[re.escape(m.group(0))], value)
return 'SELECT * FROM registry WHERE path LIKE \'%s\';' % value
def handle_regkeyvalue(value):
key, value = value.split('|')
rep = {'HKCU': 'HKEY_USERS\\%', 'HKLM': 'HKEY_LOCAL_MACHINE'}
@ -41,18 +43,23 @@ def handle_regkeyvalue(value):
key = pattern.sub(lambda m: rep[re.escape(m.group(0))], key)
return 'SELECT * FROM registry WHERE path LIKE \'%s\' AND data LIKE \'%s\';' % (key, value)
def handle_mutex(value):
return 'SELECT * FROM winbaseobj WHERE object_name LIKE \'%s\';' % value
def handle_service(value):
return 'SELECT * FROM services WHERE display_name LIKE \'%s\' OR name like \'%s\';' % (value, value)
def handle_yara(value):
return 'not implemented yet, not sure it\'s easily feasible w/o dropping the sig on the hosts first'
def handle_scheduledtask(value):
return 'SELECT * FROM scheduled_tasks WHERE name LIKE \'%s\';' % value
handlers = {
'regkey': handle_regkey,
'regkey|value': handle_regkeyvalue,
@ -62,6 +69,7 @@ handlers = {
'yara': handle_yara
}
def handler(q=False):
if q is False:
return False

7
misp_modules/modules/export_mod/testexport.py
View File

@ -1,13 +1,12 @@
import json
import base64
import csv
misperrors = {'error': 'Error'}
userConfig = {
};
}
moduleconfig = []
@ -28,8 +27,8 @@ def handler(q=False):
if q is False:
return False
r = {'results': []}
result = json.loads(q)
output = ''; # Insert your magic here!
result = json.loads(q) # noqa
output = '' # Insert your magic here!
r = {"data": base64.b64encode(output.encode('utf-8')).decode('utf-8')}
return r

2
misp_modules/modules/export_mod/threatStream_misp_export.py
View File

@ -49,10 +49,8 @@ def handler(q=False):
if q is False or not q:
return False
request = json.loads(q)
response = io.StringIO()
writer = csv.DictWriter(response, fieldnames=["value", "itype", "tags"])
writer.writeheader()

2
misp_modules/modules/import_mod/__init__.py
View File

@ -1,3 +1,3 @@
from . import _vmray
from . import _vmray # noqa
__all__ = ['vmray_import', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport']

22
misp_modules/modules/import_mod/csvimport.py
View File

@ -1,5 +1,7 @@
# -*- coding: utf-8 -*-
import json, os, base64
import json
import os
import base64
import pymisp
misperrors = {'error': 'Error'}
@ -21,6 +23,7 @@ duplicatedFields = {'mispType': {'mispComment': 'comment'},
attributesFields = ['type', 'value', 'category', 'to_ids', 'comment', 'distribution']
delimiters = [',', ';', '|', '/', '\t', ' ']
class CsvParser():
def __init__(self, header, has_header):
self.header = header
@ -32,17 +35,17 @@ class CsvParser():
return_data = []
if self.fields_number == 1:
for line in data:
l = line.split('#')[0].strip()
if l:
return_data.append(l)
line = line.split('#')[0].strip()
if line:
return_data.append(line)
self.delimiter = None
else:
self.delimiter_count = dict([(d, 0) for d in delimiters])
for line in data:
l = line.split('#')[0].strip()
if l:
self.parse_delimiter(l)
return_data.append(l)
line = line.split('#')[0].strip()
if line:
self.parse_delimiter(line)
return_data.append(line)
# find which delimiter is used
self.delimiter = self.find_delimiter()
self.data = return_data[1:] if self.has_header else return_data
@ -115,6 +118,7 @@ class CsvParser():
# return list of indexes of the misp types, list of the misp types, remaining fields that will be attribute fields
return list2pop, misp, list(reversed(head))
def handler(q=False):
if q is False:
return False
@ -138,6 +142,7 @@ def handler(q=False):
r = {'results': csv_parser.attributes}
return r
def introspection():
modulesetup = {}
try:
@ -152,6 +157,7 @@ def introspection():
pass
return modulesetup
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

6
misp_modules/modules/import_mod/cuckooimport.py
View File

@ -1,6 +1,4 @@
import json
import logging
import sys
import base64
misperrors = {'error': 'Error'}
@ -13,6 +11,7 @@ moduleinfo = {'version': '0.1', 'author': 'Victor van der Stoep',
moduleconfig = []
def handler(q=False):
# Just in case we have no data
if q is False:
@ -63,6 +62,7 @@ def handler(q=False):
# Return
return r
def processSummary(r, summary):
r["results"].append({
"values": summary['mutexes'],
@ -71,6 +71,7 @@ def processSummary(r, summary):
"comment": "Cuckoo analysis: Observed mutexes"
})
def processVT(r, virustotal):
category = "Antivirus detection"
comment = "VirusTotal analysis"
@ -187,6 +188,7 @@ def version():
moduleinfo['config'] = moduleconfig
return moduleinfo
if __name__ == '__main__':
x = open('test.json', 'r')
q = []

7
misp_modules/modules/import_mod/email_import.py
View File

@ -115,7 +115,7 @@ def handler(q=False):
email_targets = set()
for rec in received:
try:
email_check = re.search("for\s(.*@.*);", rec).group(1)
email_check = re.search(r"for\s(.*@.*);", rec).group(1)
email_check = email_check.strip(' <>')
email_targets.add(parseaddr(email_check)[1])
except (AttributeError):
@ -166,7 +166,7 @@ def handler(q=False):
for ext in zipped_files:
if filename.endswith(ext) is True:
zipped_filetype = True
if zipped_filetype == False:
if not zipped_filetype:
try:
attachment_files += get_zipped_contents(filename, attachment_data)
except RuntimeError: # File is encrypted with a password
@ -294,7 +294,7 @@ def get_zip_passwords(message):
# Grab any strings that are marked off by special chars
marking_chars = [["\'", "\'"], ['"', '"'], ['[', ']'], ['(', ')']]
for char_set in marking_chars:
regex = re.compile("""\{0}([^\{1}]*)\{1}""".format(char_set[0], char_set[1]))
regex = re.compile(r"""\{0}([^\{1}]*)\{1}""".format(char_set[0], char_set[1]))
marked_off = re.findall(regex, raw_text)
possible_passwords += marked_off
@ -397,6 +397,7 @@ def version():
moduleinfo['config'] = moduleconfig
return moduleinfo
if __name__ == '__main__':
with open('tests/test_no_attach.eml', 'r') as email_file:
handler(q=email_file.read())

11
misp_modules/modules/import_mod/goamlimport.py
View File

@ -1,6 +1,7 @@
import json, datetime, time, base64
import json
import time
import base64
import xml.etree.ElementTree as ET
from collections import defaultdict
from pymisp import MISPEvent, MISPObject
misperrors = {'error': 'Error'}
@ -73,6 +74,7 @@ goAMLmapping = {'from_account': t_account_mapping, 'to_account': t_account_mappi
nodes_to_ignore = ['addresses', 'signatory']
relationship_to_keep = ['signatory', 't_from', 't_from_my_client', 't_to', 't_to_my_client', 'address']
class GoAmlParser():
def __init__(self):
self.misp_event = MISPEvent()
@ -145,6 +147,7 @@ class GoAmlParser():
to_country_attribute = {'object_relation': 'to-country', 'value': to_country}
misp_object.add_attribute(**to_country_attribute)
def handler(q=False):
if q is False:
return False
@ -157,16 +160,18 @@ def handler(q=False):
aml_parser = GoAmlParser()
try:
aml_parser.read_xml(data)
except:
except Exception:
misperrors['error'] = "Impossible to read XML data"
return misperrors
aml_parser.parse_xml()
r = {'results': [obj.to_json() for obj in aml_parser.misp_event.objects]}
return r
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

6
misp_modules/modules/import_mod/mispjson.py
View File

@ -2,7 +2,7 @@ import json
import base64
misperrors = {'error': 'Error'}
userConfig = { };
userConfig = {}
inputSource = ['file']
@ -32,10 +32,11 @@ def handler(q=False):
if a.get("data"):
tmp["data"] = a["data"]
r['results'].append(tmp)
except:
except Exception:
pass
return r
def introspection():
modulesetup = {}
try:
@ -55,6 +56,7 @@ def version():
moduleinfo['config'] = moduleconfig
return moduleinfo
if __name__ == '__main__':
x = open('test.json', 'r')
r = handler(q=x.read())

4
misp_modules/modules/import_mod/ocr.py
View File

@ -14,7 +14,7 @@ ch.setFormatter(formatter)
log.addHandler(ch)
misperrors = {'error': 'Error'}
userConfig = {};
userConfig = {}
inputSource = ['file']
@ -78,7 +78,6 @@ def handler(q=False):
misperrors['error'] = "Corrupt or not an image file."
return misperrors
ocrized = image_to_string(im)
freetext = {}
@ -107,6 +106,7 @@ def version():
moduleinfo['config'] = moduleconfig
return moduleinfo
if __name__ == '__main__':
x = open('test.json', 'r')
handler(q=x.read())

5
misp_modules/modules/import_mod/testimport.py
View File

@ -1,6 +1,5 @@
import json
import base64
import csv
misperrors = {'error': 'Error'}
userConfig = {
@ -22,7 +21,7 @@ userConfig = {
'type': 'Integer',
'message': 'Column number used for comment'
}
};
}
inputSource = ['file', 'paste']
@ -39,7 +38,7 @@ def handler(q=False):
r = {'results': []}
request = json.loads(q)
request["data"] = base64.b64decode(request["data"])
fields = ["value", "category", "type", "comment"]
# fields = ["value", "category", "type", "comment"]
r = {"results": [{"values": ["192.168.56.1"], "types":["ip-src"], "categories": ["Network activity"]}]}
return r

4
misp_modules/modules/import_mod/threatanalyzer_import.py
View File

@ -90,7 +90,7 @@ def handler(q=False):
'values': sample_filename,
'data': base64.b64encode(file_data).decode(),
'type': 'malware-sample', 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': True, 'comment': ''})
except Exception as e:
except Exception:
# no 'sample' in archive, might be an url analysis, just ignore
pass
@ -171,7 +171,7 @@ def process_analysis_json(analysis_json):
if ':' in val:
try:
val_port = int(val.split(':')[1])
except ValueError as e:
except ValueError:
val_port = False
val_hostname = cleanup_hostname(val.split(':')[0])
val_ip = cleanup_ip(val.split(':')[0])

2
misp_modules/modules/import_mod/vmray_import.py
View File

@ -117,7 +117,7 @@ def handler(q=False):
else:
misperrors['error'] = "Unable to fetch sample id %u" % (sample_id)
return misperrors
except:
except Exception:
misperrors['error'] = "Unable to access VMRay API"
return misperrors
else: