Browse Source

fix: Make pep8 happy

pipenv
Raphaël Vinot 3 years ago
parent
commit
8fc5b1fd1f
  1. 3
      .travis.yml
  2. 11
      misp_modules/__init__.py
  3. 3
      misp_modules/helpers/cache.py
  4. 6
      misp_modules/modules/__init__.py
  5. 2
      misp_modules/modules/expansion/__init__.py
  6. 65
      misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py
  7. 6
      misp_modules/modules/expansion/asn_history.py
  8. 57
      misp_modules/modules/expansion/btc_steroids.py
  9. 40
      misp_modules/modules/expansion/countrycode.py
  10. 7
      misp_modules/modules/expansion/dbl_spamhaus.py
  11. 2
      misp_modules/modules/expansion/dns.py
  12. 6
      misp_modules/modules/expansion/farsight_passivedns.py
  13. 4
      misp_modules/modules/expansion/geoip_country.py
  14. 2
      misp_modules/modules/expansion/hashdd.py
  15. 6
      misp_modules/modules/expansion/ipasn.py
  16. 8
      misp_modules/modules/expansion/iprep.py
  17. 50
      misp_modules/modules/expansion/macaddress_io.py
  18. 12
      misp_modules/modules/expansion/onyphe.py
  19. 28
      misp_modules/modules/expansion/onyphe_full.py
  20. 16
      misp_modules/modules/expansion/otx.py
  21. 2
      misp_modules/modules/expansion/passivetotal.py
  22. 121
      misp_modules/modules/expansion/rbl.py
  23. 15
      misp_modules/modules/expansion/reversedns.py
  24. 6
      misp_modules/modules/expansion/securitytrails.py
  25. 11
      misp_modules/modules/expansion/sigma_queries.py
  26. 3
      misp_modules/modules/expansion/sigma_syntax_validator.py
  27. 5
      misp_modules/modules/expansion/stix2_pattern_syntax_validator.py
  28. 20
      misp_modules/modules/expansion/threatcrowd.py
  29. 34
      misp_modules/modules/expansion/threatminer.py
  30. 4
      misp_modules/modules/expansion/urlscan.py
  31. 4
      misp_modules/modules/expansion/virustotal.py
  32. 10
      misp_modules/modules/expansion/vmray_submit.py
  33. 36
      misp_modules/modules/expansion/vulndb.py
  34. 9
      misp_modules/modules/expansion/vulners.py
  35. 14
      misp_modules/modules/expansion/wiki.py
  36. 140
      misp_modules/modules/expansion/xforceexchange.py
  37. 4
      misp_modules/modules/expansion/yara_query.py
  38. 3
      misp_modules/modules/expansion/yara_syntax_validator.py
  39. 3
      misp_modules/modules/export_mod/__init__.py
  40. 70
      misp_modules/modules/export_mod/cef_export.py
  41. 6
      misp_modules/modules/export_mod/goamlexport.py
  42. 143
      misp_modules/modules/export_mod/liteexport.py
  43. 24
      misp_modules/modules/export_mod/osqueryexport.py
  44. 2
      misp_modules/modules/export_mod/pdfexport.py
  45. 9
      misp_modules/modules/export_mod/testexport.py
  46. 2
      misp_modules/modules/export_mod/threatStream_misp_export.py
  47. 2
      misp_modules/modules/import_mod/__init__.py
  48. 34
      misp_modules/modules/import_mod/csvimport.py
  49. 150
      misp_modules/modules/import_mod/cuckooimport.py
  50. 7
      misp_modules/modules/import_mod/email_import.py
  51. 21
      misp_modules/modules/import_mod/goamlimport.py
  52. 34
      misp_modules/modules/import_mod/mispjson.py
  53. 10
      misp_modules/modules/import_mod/ocr.py
  54. 43
      misp_modules/modules/import_mod/testimport.py
  55. 8
      misp_modules/modules/import_mod/threatanalyzer_import.py
  56. 8
      misp_modules/modules/import_mod/vmray_import.py

3
.travis.yml

@ -13,7 +13,7 @@ python:
- "3.7-dev"
install:
- pip install -U nose codecov pytest
- pip install -U nose codecov pytest flake8
- pip install -U -r REQUIREMENTS
- pip install .
@ -30,6 +30,7 @@ script:
- sleep 5
- nosetests --with-coverage --cover-package=misp_modules
- kill -s INT $pid
- flake8 --ignore=E501,W503 misp_modules
after_success:
- coverage combine .coverage*

11
misp_modules/__init__.py

@ -38,14 +38,14 @@ from tornado.concurrent import run_on_executor
from concurrent.futures import ThreadPoolExecutor
try:
from .modules import *
from .modules import * # noqa
HAS_PACKAGE_MODULES = True
except Exception as e:
print(e)
HAS_PACKAGE_MODULES = False
try:
from .helpers import *
from .helpers import * # noqa
HAS_PACKAGE_HELPERS = True
except Exception as e:
print(e)
@ -148,7 +148,7 @@ def load_package_modules():
mhandlers = {}
modules = []
for path, module in sys.modules.items():
r = re.findall("misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
r = re.findall(r"misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
if r and len(r[0]) == 2:
moduletype, modulename = r[0]
mhandlers[modulename] = module
@ -159,6 +159,9 @@ def load_package_modules():
class ListModules(tornado.web.RequestHandler):
global loaded_modules
global mhandlers
def get(self):
ret = []
for module in loaded_modules:
@ -238,7 +241,7 @@ def main():
for module in args.m:
mispmod = importlib.import_module(module)
mispmod.register(mhandlers, loaded_modules)
service = [(r'/modules', ListModules), (r'/query', QueryModule)]
application = tornado.web.Application(service)

3
misp_modules/helpers/cache.py

@ -33,7 +33,7 @@ def selftest(enable=True):
r = redis.StrictRedis(host=hostname, port=port, db=db)
try:
r.ping()
except:
except Exception:
return 'Redis not running or not installed. Helper will be disabled.'
@ -62,6 +62,7 @@ def flush():
returncode = r.flushdb()
return returncode
if __name__ == "__main__":
import sys
if selftest() is not None:

6
misp_modules/modules/__init__.py

@ -1,3 +1,3 @@
from .expansion import *
from .import_mod import *
from .export_mod import *
from .expansion import * # noqa
from .import_mod import * # noqa
from .export_mod import * # noqa

2
misp_modules/modules/expansion/__init__.py

@ -1,4 +1,4 @@
from . import _vmray
from . import _vmray # noqa
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl',
'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi',

65
misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py

@ -47,9 +47,11 @@ options = None
locale.setlocale(locale.LC_ALL, '')
class QueryError(Exception):
pass
class DnsdbClient(object):
def __init__(self, server, apikey, limit=None, http_proxy=None, https_proxy=None):
self.server = server
@ -81,7 +83,6 @@ class DnsdbClient(object):
return self._query(path, before, after)
def _query(self, path, before=None, after=None):
res = []
url = '%s/lookup/%s' % (self.server, path)
params = {}
@ -120,12 +121,15 @@ class DnsdbClient(object):
except (HTTPError, URLError) as e:
raise QueryError(str(e), sys.exc_traceback)
def quote(path):
return urllib_quote(path, safe='')
def sec_to_text(ts):
return time.strftime('%Y-%m-%d %H:%M:%S -0000', time.gmtime(ts))
def rrset_to_text(m):
s = StringIO()
@ -155,9 +159,11 @@ def rrset_to_text(m):
finally:
s.close()
def rdata_to_text(m):
return '%s IN %s %s' % (m['rrname'], m['rrtype'], m['rdata'])
def parse_config(cfg_files):
config = {}
@ -172,6 +178,7 @@ def parse_config(cfg_files):
return config
def time_parse(s):
try:
epoch = int(s)
@ -193,14 +200,15 @@ def time_parse(s):
m = re.match(r'^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$', s, re.I)
if m:
return -1*(int(m.group(1) or 0)*604800 +
int(m.group(2) or 0)*86400+
int(m.group(3) or 0)*3600+
int(m.group(4) or 0)*60+
int(m.group(5) or 0))
return -1 * (int(m.group(1) or 0) * 604800
+ int(m.group(2) or 0) * 86400
+ int(m.group(3) or 0) * 3600
+ int(m.group(4) or 0) * 60
+ int(m.group(5) or 0))
raise ValueError('Invalid time: "%s"' % s)
def epipe_wrapper(func):
def f(*args, **kwargs):
try:
@ -211,31 +219,23 @@ def epipe_wrapper(func):
raise
return f
@epipe_wrapper
def main():
global cfg
global options
parser = optparse.OptionParser(epilog='Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)')
parser.add_option('-c', '--config', dest='config',
help='config file', action='append')
parser.add_option('-r', '--rrset', dest='rrset', type='string',
help='rrset <ONAME>[/<RRTYPE>[/BAILIWICK]]')
parser.add_option('-n', '--rdataname', dest='rdata_name', type='string',
help='rdata name <NAME>[/<RRTYPE>]')
parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string',
help='rdata ip <IPADDRESS|IPRANGE|IPNETWORK>')
parser.add_option('-t', '--rrtype', dest='rrtype', type='string',
help='rrset or rdata rrtype')
parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string',
help='rrset bailiwick')
parser.add_option('-c', '--config', dest='config', help='config file', action='append')
parser.add_option('-r', '--rrset', dest='rrset', type='string', help='rrset <ONAME>[/<RRTYPE>[/BAILIWICK]]')
parser.add_option('-n', '--rdataname', dest='rdata_name', type='string', help='rdata name <NAME>[/<RRTYPE>]')
parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string', help='rdata ip <IPADDRESS|IPRANGE|IPNETWORK>')
parser.add_option('-t', '--rrtype', dest='rrtype', type='string', help='rrset or rdata rrtype')
parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string', help='rrset bailiwick')
parser.add_option('-s', '--sort', dest='sort', type='string', help='sort key')
parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False,
help='reverse sort')
parser.add_option('-j', '--json', dest='json', action='store_true', default=False,
help='output in JSON format')
parser.add_option('-l', '--limit', dest='limit', type='int', default=0,
help='limit number of results')
parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False, help='reverse sort')
parser.add_option('-j', '--json', dest='json', action='store_true', default=False, help='output in JSON format')
parser.add_option('-l', '--limit', dest='limit', type='int', default=0, help='limit number of results')
parser.add_option('', '--before', dest='before', type='string', help='only output results seen before this time')
parser.add_option('', '--after', dest='after', type='string', help='only output results seen after this time')
@ -263,20 +263,20 @@ def main():
print(str(e), file=sys.stderr)
sys.exit(1)
if not 'DNSDB_SERVER' in cfg:
if 'DNSDB_SERVER' not in cfg:
cfg['DNSDB_SERVER'] = DEFAULT_DNSDB_SERVER
if not 'HTTP_PROXY' in cfg:
if 'HTTP_PROXY' not in cfg:
cfg['HTTP_PROXY'] = DEFAULT_HTTP_PROXY
if not 'HTTPS_PROXY' in cfg:
if 'HTTPS_PROXY' not in cfg:
cfg['HTTPS_PROXY'] = DEFAULT_HTTPS_PROXY
if not 'APIKEY' in cfg:
if 'APIKEY' not in cfg:
sys.stderr.write('dnsdb_query: APIKEY not defined in config file\n')
sys.exit(1)
client = DnsdbClient(cfg['DNSDB_SERVER'], cfg['APIKEY'],
limit=options.limit,
http_proxy=cfg['HTTP_PROXY'],
https_proxy=cfg['HTTPS_PROXY'])
limit=options.limit,
http_proxy=cfg['HTTP_PROXY'],
https_proxy=cfg['HTTPS_PROXY'])
if options.rrset:
if options.rrtype or options.bailiwick:
qargs = (options.rrset, options.rrtype, options.bailiwick)
@ -307,7 +307,7 @@ def main():
if options.sort:
results = list(results)
if len(results) > 0:
if not options.sort in results[0]:
if options.sort not in results[0]:
sort_keys = results[0].keys()
sort_keys.sort()
sys.stderr.write('dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' % (options.sort, ', '.join(sort_keys)))
@ -319,5 +319,6 @@ def main():
print(e.message, file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
main()

6
misp_modules/modules/expansion/asn_history.py

@ -22,9 +22,9 @@ def handler(q=False):
misperrors['error'] = "Unsupported attributes type"
return misperrors
if not request.get('config') and not (request['config'].get('host') and
request['config'].get('port') and
request['config'].get('db')):
if not request.get('config') and not (request['config'].get('host')
and request['config'].get('port')
and request['config'].get('db')):
misperrors['error'] = 'ASN description history configuration is missing'
return misperrors

57
misp_modules/modules/expansion/btc_steroids.py

@ -12,24 +12,25 @@ moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
moduleconfig = []
blockchain_firstseen='https://blockchain.info/q/addressfirstseen/'
blockchain_balance='https://blockchain.info/q/addressbalance/'
blockchain_totalreceived='https://blockchain.info/q/getreceivedbyaddress/'
blockchain_all='https://blockchain.info/rawaddr/'
converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts='
blockchain_firstseen = 'https://blockchain.info/q/addressfirstseen/'
blockchain_balance = 'https://blockchain.info/q/addressbalance/'
blockchain_totalreceived = 'https://blockchain.info/q/getreceivedbyaddress/'
blockchain_all = 'https://blockchain.info/rawaddr/{}?filter=5{}'
converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts={}'
converter_rls = 'https://min-api.cryptocompare.com/stats/rate/limit'
result_text = ""
g_rate_limit = 300
start_time = 0
conversion_rates = {}
def get_consumption(output=False):
try:
req = requests.get(converter_rls)
jreq = req.json()
minute = str(jreq['Data']['calls_left']['minute'])
hour = str(jreq['Data']['calls_left']['hour'])
except:
hour = str(jreq['Data']['calls_left']['hour'])
except Exception:
minute = str(-1)
hour = str(-1)
# Debug out for the console
@ -53,20 +54,20 @@ def convert(btc, timestamp):
minute, hour = get_consumption()
g_rate_limit -= 1
now = time.time()
delta = now - start_time
#print(g_rate_limit)
# delta = now - start_time
# print(g_rate_limit)
if g_rate_limit <= 10:
minute, hour = get_consumption(output=True)
if int(minute) <= 10:
#print(minute)
#get_consumption(output=True)
# print(minute)
# get_consumption(output=True)
time.sleep(3)
else:
mprint(minute)
start_time = time.time()
g_rate_limit = int(minute)
try:
req = requests.get(converter+str(timestamp))
req = requests.get(converter.format(timestamp))
jreq = req.json()
usd = jreq['BTC']['USD']
eur = jreq['BTC']['EUR']
@ -78,7 +79,7 @@ def convert(btc, timestamp):
# Actually convert and return the values
u = usd * btc
e = eur * btc
return u,e
return u, e
def mprint(input):
@ -90,8 +91,8 @@ def mprint(input):
def handler(q=False):
global result_text
global conversion_rates
start_time = time.time()
now = time.time()
# start_time = time.time()
# now = time.time()
if q is False:
return False
request = json.loads(q)
@ -107,13 +108,13 @@ def handler(q=False):
mprint("\nAddress:\t" + btc)
try:
req = requests.get(blockchain_all+btc+"?limit=50&filter=5")
req = requests.get(blockchain_all.format(btc, "&limit=50"))
jreq = req.json()
except Exception as e:
#print(e)
except Exception:
# print(e)
print(req.text)
result_text = ""
sys.exit(1)
sys.exit(1)
n_tx = jreq['n_tx']
balance = float(jreq['final_balance'] / 100000000)
@ -130,11 +131,11 @@ def handler(q=False):
i = 0
while i < n_tx:
if click is False:
req = requests.get(blockchain_all+btc+"?limit=5&offset="+str(i)+"&filter=5")
req = requests.get(blockchain_all.format(btc, "&limit=5&offset={}".format(i)))
if n_tx > 5:
n_tx = 5
else:
req = requests.get(blockchain_all+btc+"?limit=50&offset="+str(i)+"&filter=5")
req = requests.get(blockchain_all.format(btc, "&limit=50&offset={}".format(i)))
jreq = req.json()
if jreq['txs']:
for transactions in jreq['txs']:
@ -144,8 +145,8 @@ def handler(q=False):
script_old = tx['script']
if tx['prev_out']['value'] != 0 and tx['prev_out']['addr'] == btc:
datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time'])))
value = float(tx['prev_out']['value'] / 100000000 )
u,e = convert(value, transactions['time'])
value = float(tx['prev_out']['value'] / 100000000)
u, e = convert(value, transactions['time'])
mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0'))
if script_old != tx['script']:
i += 1
@ -153,19 +154,19 @@ def handler(q=False):
sum_counter += 1
sum += value
if sum_counter > 1:
u,e = convert(sum, transactions['time'])
u, e = convert(sum, transactions['time'])
mprint("\t\t\t\t\t----------------------------------------------")
mprint("#" + str(n_tx - i) + "\t\t\t\t Sum:\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR\n".format(sum, u, e).rstrip('0'))
for tx in transactions['out']:
if tx['value'] != 0 and tx['addr'] == btc:
datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time'])))
value = float(tx['value'] / 100000000 )
u,e = convert(value, transactions['time'])
value = float(tx['value'] / 100000000)
u, e = convert(value, transactions['time'])
mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t {0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0'))
#i += 1
# i += 1
i += 1
r = {
r = {
'results': [
{
'types': ['text'],

40
misp_modules/modules/expansion/countrycode.py

@ -12,20 +12,21 @@ moduleinfo = {'version': '1', 'author': 'Hannah Ward',
# config fields that your code expects from the site admin
moduleconfig = []
common_tlds = {"com":"Commercial (Worldwide)",
"org":"Organisation (Worldwide)",
"net":"Network (Worldwide)",
"int":"International (Worldwide)",
"edu":"Education (Usually USA)",
"gov":"Government (USA)"
}
common_tlds = {"com": "Commercial (Worldwide)",
"org": "Organisation (Worldwide)",
"net": "Network (Worldwide)",
"int": "International (Worldwide)",
"edu": "Education (Usually USA)",
"gov": "Government (USA)"
}
codes = False
def handler(q=False):
global codes
if not codes:
codes = requests.get("http://www.geognos.com/api/en/countries/info/all.json").json()
codes = requests.get("http://www.geognos.com/api/en/countries/info/all.json").json()
if q is False:
return False
request = json.loads(q)
@ -36,18 +37,18 @@ def handler(q=False):
# Check if it's a common, non country one
if ext in common_tlds.keys():
val = common_tlds[ext]
val = common_tlds[ext]
else:
# Retrieve a json full of country info
if not codes["StatusMsg"] == "OK":
val = "Unknown"
else:
# Find our code based on TLD
codes = codes["Results"]
for code in codes.keys():
if codes[code]["CountryCodes"]["tld"] == ext:
val = codes[code]["Name"]
r = {'results': [{'types':['text'], 'values':[val]}]}
# Retrieve a json full of country info
if not codes["StatusMsg"] == "OK":
val = "Unknown"
else:
# Find our code based on TLD
codes = codes["Results"]
for code in codes.keys():
if codes[code]["CountryCodes"]["tld"] == ext:
val = codes[code]["Name"]
r = {'results': [{'types': ['text'], 'values':[val]}]}
return r
@ -58,4 +59,3 @@ def introspection():
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

7
misp_modules/modules/expansion/dbl_spamhaus.py

@ -1,6 +1,5 @@
import json
import datetime
from collections import defaultdict
import sys
try:
import dns.resolver
@ -30,12 +29,14 @@ dbl_mapping = {'127.0.1.2': 'spam domain',
'127.0.1.106': 'abused legit botnet C&C',
'127.0.1.255': 'IP queries prohibited!'}
def fetch_requested_value(request):
for attribute_type in mispattributes['input']:
if request.get(attribute_type):
return request[attribute_type].split('|')[0]
return None
def handler(q=False):
if q is False:
return False
@ -52,9 +53,11 @@ def handler(q=False):
result = str(e)
return {'results': [{'types': mispattributes.get('output'), 'values': result}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

2
misp_modules/modules/expansion/dns.py

@ -43,7 +43,7 @@ def handler(q=False):
except dns.exception.Timeout:
misperrors['error'] = "Timeout"
return misperrors
except:
except Exception:
misperrors['error'] = "DNS resolving error"
return misperrors

6
misp_modules/modules/expansion/farsight_passivedns.py

@ -51,7 +51,7 @@ def lookup_name(client, name):
for i in item.get('rdata'):
# grab email field and replace first dot by @ to convert to an email address
yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1))
except QueryError as e:
except QueryError:
pass
try:
@ -59,7 +59,7 @@ def lookup_name(client, name):
for item in res:
if item.get('rrtype') in ['A', 'AAAA', 'CNAME']:
yield(item.get('rrname').rstrip('.'))
except QueryError as e:
except QueryError:
pass
@ -68,7 +68,7 @@ def lookup_ip(client, ip):
res = client.query_rdata_ip(ip)
for item in res:
yield(item['rrname'].rstrip('.'))
except QueryError as e:
except QueryError:
pass

4
misp_modules/modules/expansion/geoip_country.py

@ -27,7 +27,7 @@ try:
config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'geoip_country.cfg'))
gi = pygeoip.GeoIP(config.get('GEOIP', 'database'))
enabled = True
except:
except Exception:
enabled = False
@ -49,7 +49,7 @@ def handler(q=False):
try:
answer = gi.country_code_by_addr(toquery)
except:
except Exception:
misperrors['error'] = "GeoIP resolving error"
return misperrors

2
misp_modules/modules/expansion/hashdd.py

@ -20,7 +20,7 @@ def handler(q=False):
if v is None:
misperrors['error'] = 'Hash value is missing.'
return misperrors
r = requests.post(hashddapi_url, data={'hash':v})
r = requests.post(hashddapi_url, data={'hash': v})
if r.status_code == 200:
state = json.loads(r.text)
if state:

6
misp_modules/modules/expansion/ipasn.py

@ -24,9 +24,9 @@ def handler(q=False):
misperrors['error'] = "Unsupported attributes type"
return misperrors
if not request.get('config') and not (request['config'].get('host') and
request['config'].get('port') and
request['config'].get('db')):
if not request.get('config') and not (request['config'].get('host')
and request['config'].get('port')
and request['config'].get('db')):
misperrors['error'] = 'IP ASN history configuration is missing'
return misperrors

8
misp_modules/modules/expansion/iprep.py

@ -45,7 +45,7 @@ def parse_iprep(ip, api):
url = 'https://www.packetmail.net/iprep.php/%s' % ip
try:
data = requests.get(url, params={'apikey': api}).json()
except:
except Exception:
return ['Error pulling data'], rep
# print '%s' % data
for name, val in data.items():
@ -71,11 +71,11 @@ def parse_iprep(ip, api):
misp_val = context
full_text += '\n%s' % context
misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen'])
rep.append({'types': mispattributes['output'], 'categories':['External analysis'], 'values': misp_val, 'comment': misp_comment})
except:
rep.append({'types': mispattributes['output'], 'categories': ['External analysis'], 'values': misp_val, 'comment': misp_comment})
except Exception:
err.append('Error parsing source: %s' % name)
rep.append({'types': ['freetext'], 'values': full_text , 'comment': 'Free text import of IPRep'})
rep.append({'types': ['freetext'], 'values': full_text, 'comment': 'Free text import of IPRep'})
return err, rep

50
misp_modules/modules/expansion/macaddress_io.py

@ -86,32 +86,30 @@ def handler(q=False):
response.block_details.date_updated.strftime('%d %B %Y') if response.block_details.date_updated else None
results = {
'results': [
{'types': ['text'], 'values':
{
# Mac address details
'Valid MAC address': "True" if response.mac_address_details.is_valid else "False",
'Transmission type': response.mac_address_details.transmission_type,
'Administration type': response.mac_address_details.administration_type,
# Vendor details
'OUI': response.vendor_details.oui,
'Vendor details are hidden': "True" if response.vendor_details.is_private else "False",
'Company name': response.vendor_details.company_name,
'Company\'s address': response.vendor_details.company_address,
'County code': response.vendor_details.country_code,
# Block details
'Block found': "True" if response.block_details.block_found else "False",
'The left border of the range': response.block_details.border_left,
'The right border of the range': response.block_details.border_right,
'The total number of MAC addresses in this range': response.block_details.block_size,
'Assignment block size': response.block_details.assignment_block_size,
'Date when the range was allocated': date_created,
'Date when the range was last updated': date_updated
}
}
]
'results':
[{'types': ['text'], 'values':
{
# Mac address details
'Valid MAC address': "True" if response.mac_address_details.is_valid else "False",
'Transmission type': response.mac_address_details.transmission_type,
'Administration type': response.mac_address_details.administration_type,
# Vendor details
'OUI': response.vendor_details.oui,
'Vendor details are hidden': "True" if response.vendor_details.is_private else "False",
'Company name': response.vendor_details.company_name,
'Company\'s address': response.vendor_details.company_address,
'County code': response.vendor_details.country_code,
# Block details
'Block found': "True" if response.block_details.block_found else "False",
'The left border of the range': response.block_details.border_left,
'The right border of the range': response.block_details.border_right,
'The total number of MAC addresses in this range': response.block_details.block_size,
'Assignment block size': response.block_details.assignment_block_size,
'Date when the range was allocated': date_created,
'Date when the range was last updated': date_updated
}}]
}
return results

12
misp_modules/modules/expansion/onyphe.py

@ -1,4 +1,3 @@
import json
# -*- coding: utf-8 -*-
import json
@ -9,7 +8,8 @@ except ImportError:
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], 'output': ['hostname', 'domain', 'ip-src', 'ip-dst','url']}
mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']}
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
'description': 'Query on Onyphe',
@ -54,7 +54,7 @@ def handle_expansion(api, ip, misperrors):
misperrors['error'] = result['message']
return misperrors
categories = list(set([item['@category'] for item in result['results']]))
# categories = list(set([item['@category'] for item in result['results']]))
result_filtered = {"results": []}
urls_pasties = []
@ -72,9 +72,9 @@ def handle_expansion(api, ip, misperrors):
os_target = r['os']
if os_target != 'Unknown':
os_list.append(r['os'])
elif r['@category'] == 'resolver' and r['type'] =='reverse':
elif r['@category'] == 'resolver' and r['type'] == 'reverse':
domains_resolver.append(r['reverse'])
elif r['@category'] == 'resolver' and r['type'] =='forward':
elif r['@category'] == 'resolver' and r['type'] == 'forward':
domains_forward.append(r['forward'])
result_filtered['results'].append({'types': ['url'], 'values': urls_pasties,
@ -90,7 +90,7 @@ def handle_expansion(api, ip, misperrors):
result_filtered['results'].append({'types': ['domain'],
'values': list(set(domains_resolver)),
'categories': ['Network activity'],
'comment': 'resolver to %s' % ip })
'comment': 'resolver to %s' % ip})
result_filtered['results'].append({'types': ['domain'],
'values': list(set(domains_forward)),

28
misp_modules/modules/expansion/onyphe_full.py

@ -1,4 +1,3 @@
import json
# -*- coding: utf-8 -*-
import json
@ -10,7 +9,7 @@ except ImportError:
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
'output': ['hostname', 'domain', 'ip-src', 'ip-dst','url']}
'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']}
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
@ -38,10 +37,10 @@ def handler(q=False):
ip = ''
if request.get('ip-src'):
ip = request['ip-src']
return handle_ip(api ,ip, misperrors)
return handle_ip(api, ip, misperrors)
elif request.get('ip-dst'):
ip = request['ip-dst']
return handle_ip(api,ip,misperrors)
return handle_ip(api, ip, misperrors)
elif request.get('domain'):
domain = request['domain']
return handle_domain(api, domain, misperrors)
@ -91,11 +90,11 @@ def handle_ip(api, ip, misperrors):
r, status_ok = expand_syscan(api, ip, misperrors)
if status_ok:
result_filtered['results'].extend(r)
result_filtered['results'].extend(r)
else:
misperrors['error'] = "Error syscan result"
misperrors['error'] = "Error syscan result"
r, status_ok = expand_pastries(api,misperrors,ip=ip)
r, status_ok = expand_pastries(api, misperrors, ip=ip)
if status_ok:
result_filtered['results'].extend(r)
@ -185,11 +184,11 @@ def expand_syscan(api, ip, misperror):
return r, status_ok
def expand_datascan(api, misperror,**kwargs):
def expand_datascan(api, misperror, **kwargs):
status_ok = False
r = []
ip = ''
query =''
# ip = ''
query = ''
asn_list = []
geoloc = []
orgs = []
@ -311,7 +310,7 @@ def expand_pastries(api, misperror, **kwargs):
query = kwargs.get('domain')
result = api.search_pastries('domain:%s' % query)
if result['status'] =='ok':
if result['status'] == 'ok':
status_ok = True
for item in result['results']:
if item['@category'] == 'pastries':
@ -328,7 +327,7 @@ def expand_pastries(api, misperror, **kwargs):
r.append({'types': ['url'],
'values': urls_pasties,
'categories': ['External analysis'],
'comment':'URLs of pasties where %s has found' % query})
'comment': 'URLs of pasties where %s has found' % query})
r.append({'types': ['domain'], 'values': list(set(domains)),
'categories': ['Network activity'],
'comment': 'Domains found in pasties of Onyphe'})
@ -340,7 +339,7 @@ def expand_pastries(api, misperror, **kwargs):
return r, status_ok
def expand_threatlist(api, misperror,**kwargs):
def expand_threatlist(api, misperror, **kwargs):
status_ok = False
r = []
@ -366,7 +365,8 @@ def expand_threatlist(api, misperror,**kwargs):
'comment': '%s is present in threatlist' % query
})
return r,status_ok
return r, status_ok
def introspection():
return mispattributes

16
misp_modules/modules/expansion/otx.py

@ -15,9 +15,10 @@ moduleinfo = {'version': '1', 'author': 'chrisdoman',
# We're not actually using the API key yet
moduleconfig = ["apikey"]
# Avoid adding windows update to enrichment etc.
def isBlacklisted(value):
blacklist = ['0.0.0.0', '8.8.8.8', '255.255.255.255', '192.168.56.' , 'time.windows.com']
blacklist = ['0.0.0.0', '8.8.8.8', '255.255.255.255', '192.168.56.', 'time.windows.com']
for b in blacklist:
if value in b:
@ -25,10 +26,12 @@ def isBlacklisted(value):
return True
def valid_ip(ip):
m = re.match(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$", ip)
return bool(m) and all(map(lambda n: 0 <= int(n) <= 255, m.groups()))
def findAll(data, keys):
a = []
if isinstance(data, dict):
@ -43,9 +46,11 @@ def findAll(data, keys):
a.extend(findAll(i, keys))
return a
def valid_email(email):
return bool(re.search(r"[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?", email))
def handler(q=False):
if q is False:
return False
@ -99,19 +104,17 @@ def getHash(_hash, key):
def getIP(ip, key):
ret = []
req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/malware/" + ip + "?limit=1000").text )
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/ip/malware/" + ip + "?limit=1000").text)
for _hash in findAll(req, "hash"):
ret.append({"types": ["sha256"], "values": [_hash]})
req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text )
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text)
for hostname in findAll(req, "hostname"):
if not isBlacklisted(hostname):
ret.append({"types": ["hostname"], "values": [hostname]})
return ret
@ -119,7 +122,7 @@ def getDomain(domain, key):
ret = []
req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text )
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text)
for _hash in findAll(req, "hash"):
ret.append({"types": ["sha256"], "values": [_hash]})
@ -144,6 +147,7 @@ def getDomain(domain, key):
return ret
def introspection():
return mispattributes

2
misp_modules/modules/expansion/passivetotal.py

@ -331,7 +331,7 @@ def handler(q=False):
output['results'] += results
else:
log.error("Unsupported query pattern issued.")
except:
except Exception:
return misperrors
return output

121
misp_modules/modules/expansion/rbl.py

@ -1,5 +1,5 @@
import json
import datetime
import sys
try:
import dns.resolver
@ -18,64 +18,65 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
moduleconfig = []
rbls = {
'spam.spamrats.com': 'http://www.spamrats.com',
'spamguard.leadmon.net': 'http://www.leadmon.net/SpamGuard/',
'rbl-plus.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
'web.dnsbl.sorbs.net': 'http://www.sorbs.net',
'ix.dnsbl.manitu.net': 'http://www.dnsbl.manitu.net',
'virus.rbl.jp': 'http://www.rbl.jp',
'dul.dnsbl.sorbs.net': 'http://www.sorbs.net',
'bogons.cymru.com': 'http://www.team-cymru.org/Services/Bogons/',
'psbl.surriel.com': 'http://psbl.surriel.com',
'misc.dnsbl.sorbs.net': 'http://www.sorbs.net',
'httpbl.abuse.ch': 'http://dnsbl.abuse.ch',
'combined.njabl.org': 'http://combined.njabl.org',
'smtp.dnsbl.sorbs.net': 'http://www.sorbs.net',
'korea.services.net': 'http://korea.services.net',
'drone.abuse.ch': 'http://dnsbl.abuse.ch',
'rbl.efnetrbl.org': 'http://rbl.efnetrbl.org',
'cbl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US',
'b.barracudacentral.org': 'http://www.barracudacentral.org/rbl/removal-request',
'bl.spamcannibal.org': 'http://www.spamcannibal.org',
'xbl.spamhaus.org': 'http://www.spamhaus.org/xbl/',
'zen.spamhaus.org': 'http://www.spamhaus.org/zen/',
'rbl.suresupport.com': 'http://suresupport.com/postmaster',
'db.wpbl.info': 'http://www.wpbl.info',
'sbl.spamhaus.org': 'http://www.spamhaus.org/sbl/',
'http.dnsbl.sorbs.net': 'http://www.sorbs.net',
'csi.cloudmark.com': 'http://www.cloudmark.com/en/products/cloudmark-sender-intelligence/index',
'rbl.interserver.net': 'http://rbl.interserver.net',
'ubl.unsubscore.com': 'http://www.lashback.com/blacklist/',
'dnsbl.sorbs.net': 'http://www.sorbs.net',
'virbl.bit.nl': 'http://virbl.bit.nl',
'pbl.spamhaus.org': 'http://www.spamhaus.org/pbl/',
'socks.dnsbl.sorbs.net': 'http://www.sorbs.net',
'short.rbl.jp': 'http://www.rbl.jp',
'dnsbl.dronebl.org': 'http://www.dronebl.org',
'blackholes.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
'truncate.gbudb.net': 'http://www.gbudb.com/truncate/index.jsp',
'dyna.spamrats.com': 'http://www.spamrats.com',
'spamrbl.imp.ch': 'http://antispam.imp.ch',
'spam.dnsbl.sorbs.net': 'http://www.sorbs.net',
'wormrbl.imp.ch': 'http://antispam.imp.ch',
'query.senderbase.org': 'http://www.senderbase.org/about',
'opm.tornevall.org': 'http://dnsbl.tornevall.org',
'netblock.pedantic.org': 'http://pedantic.org',
'access.redhawk.org': 'http://www.redhawk.org/index.php?option=com_wrapper&Itemid=33',
'cdl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US',
'multi.surbl.org': 'http://www.surbl.org',
'noptr.spamrats.com': 'http://www.spamrats.com',
'dnsbl.inps.de': 'http://dnsbl.inps.de/index.cgi?lang=en',
'bl.spamcop.net': 'http://bl.spamcop.net',
'cbl.abuseat.org': 'http://cbl.abuseat.org',
'dsn.rfc-ignorant.org': 'http://www.rfc-ignorant.org/policy-dsn.php',
'zombie.dnsbl.sorbs.net': 'http://www.sorbs.net',
'dnsbl.njabl.org': 'http://dnsbl.njabl.org',
'relays.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
'rbl.spamlab.com': 'http://tools.appriver.com/index.aspx?tool=rbl',
'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html'
'spam.spamrats.com': 'http://www.spamrats.com',
'spamguard.leadmon.net': 'http://www.leadmon.net/SpamGuard/',
'rbl-plus.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
'web.dnsbl.sorbs.net': 'http://www.sorbs.net',
'ix.dnsbl.manitu.net': 'http://www.dnsbl.manitu.net',
'virus.rbl.jp': 'http://www.rbl.jp',
'dul.dnsbl.sorbs.net': 'http://www.sorbs.net',
'bogons.cymru.com': 'http://www.team-cymru.org/Services/Bogons/',
'psbl.surriel.com': 'http://psbl.surriel.com',
'misc.dnsbl.sorbs.net': 'http://www.sorbs.net',
'httpbl.abuse.ch': 'http://dnsbl.abuse.ch',
'combined.njabl.org': 'http://combined.njabl.org',
'smtp.dnsbl.sorbs.net': 'http://www.sorbs.net',
'korea.services.net': 'http://korea.services.net',
'drone.abuse.ch': 'http://dnsbl.abuse.ch',
'rbl.efnetrbl.org': 'http://rbl.efnetrbl.org',
'cbl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US',
'b.barracudacentral.org': 'http://www.barracudacentral.org/rbl/removal-request',
'bl.spamcannibal.org': 'http://www.spamcannibal.org',
'xbl.spamhaus.org': 'http://www.spamhaus.org/xbl/',
'zen.spamhaus.org': 'http://www.spamhaus.org/zen/',
'rbl.suresupport.com': 'http://suresupport.com/postmaster',
'db.wpbl.info': 'http://www.wpbl.info',
'sbl.spamhaus.org': 'http://www.spamhaus.org/sbl/',
'http.dnsbl.sorbs.net': 'http://www.sorbs.net',
'csi.cloudmark.com': 'http://www.cloudmark.com/en/products/cloudmark-sender-intelligence/index',
'rbl.interserver.net': 'http://rbl.interserver.net',
'ubl.unsubscore.com': 'http://www.lashback.com/blacklist/',
'dnsbl.sorbs.net': 'http://www.sorbs.net',
'virbl.bit.nl': 'http://virbl.bit.nl',
'pbl.spamhaus.org': 'http://www.spamhaus.org/pbl/',
'socks.dnsbl.sorbs.net': 'http://www.sorbs.net',
'short.rbl.jp': 'http://www.rbl.jp',
'dnsbl.dronebl.org': 'http://www.dronebl.org',
'blackholes.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
'truncate.gbudb.net': 'http://www.gbudb.com/truncate/index.jsp',
'dyna.spamrats.com': 'http://www.spamrats.com',
'spamrbl.imp.ch': 'http://antispam.imp.ch',
'spam.dnsbl.sorbs.net': 'http://www.sorbs.net',
'wormrbl.imp.ch': 'http://antispam.imp.ch',
'query.senderbase.org': 'http://www.senderbase.org/about',
'opm.tornevall.org': 'http://dnsbl.tornevall.org',
'netblock.pedantic.org': 'http://pedantic.org',
'access.redhawk.org': 'http://www.redhawk.org/index.php?option=com_wrapper&Itemid=33',
'cdl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US',
'multi.surbl.org': 'http://www.surbl.org',
'noptr.spamrats.com': 'http://www.spamrats.com',
'dnsbl.inps.de': 'http://dnsbl.inps.de/index.cgi?lang=en',
'bl.spamcop.net': 'http://bl.spamcop.net',
'cbl.abuseat.org': 'http://cbl.abuseat.org',
'dsn.rfc-ignorant.org': 'http://www.rfc-ignorant.org/policy-dsn.php',
'zombie.dnsbl.sorbs.net': 'http://www.sorbs.net',
'dnsbl.njabl.org': 'http://dnsbl.njabl.org',
'relays.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
'rbl.spamlab.com': 'http://tools.appriver.com/index.aspx?tool=rbl',
'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html'
}
def handler(q=False):
if q is False:
return False
@ -89,11 +90,11 @@ def handler(q=False):
return misperrors
listed = []
info = []
ipRev = '.'.join(ip.split('.')[::-1])
ipRev = '.'.join(ip.split('.')[::-1])
for rbl in rbls:
query = '{}.{}'.format(ipRev, rbl)
try:
txt = resolver.query(query,'TXT')
txt = resolver.query(query, 'TXT')
listed.append(query)
info.append([str(t) for t in txt])
except Exception:
@ -101,9 +102,11 @@ def handler(q=False):
result = "\n".join(["{}: {}".format(l, " - ".join(i)) for l, i in zip(listed, info)])
return {'results': [{'types': mispattributes.get('output'), 'values': result}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

15
misp_modules/modules/expansion/reversedns.py

@ -1,5 +1,5 @@
import json
import dns.reversename, dns.resolver
from dns import reversename, resolver, exception
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['hostname']}
@ -12,6 +12,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
# config fields that your code expects from the site admin
moduleconfig = ['nameserver']
def handler(q=False):
if q is False:
return False
@ -26,9 +27,9 @@ def handler(q=False):
return False
# reverse lookup for ip
revname = dns.reversename.from_address(toquery)
revname = reversename.from_address(toquery)
r = dns.resolver.Resolver()
r = resolver.Resolver()
r.timeout = 2
r.lifetime = 2
@ -42,13 +43,13 @@ def handler(q=False):
try:
answer = r.query(revname, 'PTR')
except dns.resolver.NXDOMAIN:
except resolver.NXDOMAIN:
misperrors['error'] = "NXDOMAIN"
return misperrors
except dns.exception.Timeout:
except exception.Timeout:
misperrors['error'] = "Timeout"
return misperrors
except:
except Exception:
misperrors['error'] = "DNS resolving error"
return misperrors
@ -56,9 +57,11 @@ def handler(q=False):
'values':[str(answer[0])]}]}
return r
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

6
misp_modules/modules/expansion/securitytrails.py

@ -114,8 +114,7 @@ def handle_domain(api, domain, misperrors):
if r:
result_filtered['results'].extend(r)
else:
misperrors['error'] = misperrors[
'error'] + ' Error in expand History DNS'
misperrors['error'] = misperrors['error'] + ' Error in expand History DNS'
return misperrors
r, status_ok = expand_history_whois(api, domain)
@ -124,8 +123,7 @@ def handle_domain(api, domain, misperrors):
if r:
result_filtered['results'].extend(r)
else:
misperrors['error'] = misperrors['error'] + \
' Error in expand History Whois'
misperrors['error'] = misperrors['error'] + ' Error in expand History Whois'
return misperrors
return result_filtered

11
misp_modules/modules/expansion/sigma_queries.py

@ -1,4 +1,6 @@
import sys, os, io, json
import sys
import io
import json
try:
from sigma.parser import SigmaCollectionParser
from sigma.config import SigmaConfiguration
@ -13,6 +15,7 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['e
moduleconfig = []
sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'wdatp', 'splunkxml', 'arcsight', 'qualys')
def handler(q=False):
if q is False:
return False
@ -35,16 +38,18 @@ def handler(q=False):
backend.finalize()
print("#NEXT")
targets.append(t)
except:
except Exception:
continue
sys.stdout = old_stdout
results = result.getvalue()[:-5].split('#NEXT')
d_result = {t: r.strip() for t,r in zip(targets, results)}
d_result = {t: r.strip() for t, r in zip(targets, results)}