fix: Make pep8 happy

.travis.yml

@@ -13,7 +13,7 @@ python:
     - "3.7-dev"
 
 install:
-    - pip install -U nose codecov pytest
+    - pip install -U nose codecov pytest flake8
     - pip install -U -r REQUIREMENTS
     - pip install .
 
@@ -30,6 +30,7 @@ script:
     - sleep 5
     - nosetests --with-coverage --cover-package=misp_modules
     - kill -s INT $pid
+    - flake8 --ignore=E501,W503 misp_modules
 
 after_success:
     - coverage combine .coverage*

misp_modules/__init__.py

@@ -38,14 +38,14 @@ from tornado.concurrent import run_on_executor
 from concurrent.futures import ThreadPoolExecutor
 
 try:
-    from .modules import *
+    from .modules import *  # noqa
     HAS_PACKAGE_MODULES = True
 except Exception as e:
     print(e)
     HAS_PACKAGE_MODULES = False
 
 try:
-    from .helpers import *
+    from .helpers import *  # noqa
     HAS_PACKAGE_HELPERS = True
 except Exception as e:
     print(e)
@@ -148,7 +148,7 @@ def load_package_modules():
     mhandlers = {}
     modules = []
     for path, module in sys.modules.items():
-        r = re.findall("misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
+        r = re.findall(r"misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
         if r and len(r[0]) == 2:
             moduletype, modulename = r[0]
             mhandlers[modulename] = module
@@ -159,6 +159,9 @@ def load_package_modules():
 
 
 class ListModules(tornado.web.RequestHandler):
+    global loaded_modules
+    global mhandlers
+
     def get(self):
         ret = []
         for module in loaded_modules:
@@ -238,7 +241,7 @@ def main():
     for module in args.m:
         mispmod = importlib.import_module(module)
         mispmod.register(mhandlers, loaded_modules)
-        
+
     service = [(r'/modules', ListModules), (r'/query', QueryModule)]
 
     application = tornado.web.Application(service)

misp_modules/helpers/cache.py

@@ -33,7 +33,7 @@ def selftest(enable=True):
     r = redis.StrictRedis(host=hostname, port=port, db=db)
     try:
         r.ping()
-    except:
+    except Exception:
         return 'Redis not running or not installed. Helper will be disabled.'
 
 
@@ -62,6 +62,7 @@ def flush():
     returncode = r.flushdb()
     return returncode
 
+
 if __name__ == "__main__":
     import sys
     if selftest() is not None:

misp_modules/modules/__init__.py

@@ -1,3 +1,3 @@
-from .expansion import *
-from .import_mod import *
-from .export_mod import *
+from .expansion import *  # noqa
+from .import_mod import *  # noqa
+from .export_mod import *  # noqa

misp_modules/modules/expansion/__init__.py

@@ -1,4 +1,4 @@
-from . import _vmray
+from . import _vmray  # noqa
 
 __all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl',
            'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi',

misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py

@@ -47,9 +47,11 @@ options = None
 
 locale.setlocale(locale.LC_ALL, '')
 
+
 class QueryError(Exception):
     pass
 
+
 class DnsdbClient(object):
     def __init__(self, server, apikey, limit=None, http_proxy=None, https_proxy=None):
         self.server = server
@@ -81,7 +83,6 @@ class DnsdbClient(object):
         return self._query(path, before, after)
 
     def _query(self, path, before=None, after=None):
-        res = []
         url = '%s/lookup/%s' % (self.server, path)
 
         params = {}
@@ -120,12 +121,15 @@ class DnsdbClient(object):
         except (HTTPError, URLError) as e:
             raise QueryError(str(e), sys.exc_traceback)
 
+
 def quote(path):
     return urllib_quote(path, safe='')
 
+
 def sec_to_text(ts):
     return time.strftime('%Y-%m-%d %H:%M:%S -0000', time.gmtime(ts))
 
+
 def rrset_to_text(m):
     s = StringIO()
 
@@ -155,9 +159,11 @@ def rrset_to_text(m):
     finally:
         s.close()
 
+
 def rdata_to_text(m):
     return '%s IN %s %s' % (m['rrname'], m['rrtype'], m['rdata'])
 
+
 def parse_config(cfg_files):
     config = {}
 
@@ -172,6 +178,7 @@ def parse_config(cfg_files):
 
     return config
 
+
 def time_parse(s):
     try:
         epoch = int(s)
@@ -193,14 +200,15 @@ def time_parse(s):
 
     m = re.match(r'^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$', s, re.I)
     if m:
-        return -1*(int(m.group(1) or 0)*604800 +
-                int(m.group(2) or 0)*86400+
-                int(m.group(3) or 0)*3600+
-                int(m.group(4) or 0)*60+
-                int(m.group(5) or 0))
+        return -1 * (int(m.group(1) or 0) * 604800
+                     + int(m.group(2) or 0) * 86400
+                     + int(m.group(3) or 0) * 3600
+                     + int(m.group(4) or 0) * 60
+                     + int(m.group(5) or 0))
 
     raise ValueError('Invalid time: "%s"' % s)
 
+
 def epipe_wrapper(func):
     def f(*args, **kwargs):
         try:
@@ -211,31 +219,23 @@ def epipe_wrapper(func):
             raise
     return f
 
+
 @epipe_wrapper
 def main():
     global cfg
     global options
 
     parser = optparse.OptionParser(epilog='Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)')
-    parser.add_option('-c', '--config', dest='config', 
-        help='config file', action='append')
-    parser.add_option('-r', '--rrset', dest='rrset', type='string',
-        help='rrset <ONAME>[/<RRTYPE>[/BAILIWICK]]')
-    parser.add_option('-n', '--rdataname', dest='rdata_name', type='string',
-        help='rdata name <NAME>[/<RRTYPE>]')
-    parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string',
-        help='rdata ip <IPADDRESS|IPRANGE|IPNETWORK>')
-    parser.add_option('-t', '--rrtype', dest='rrtype', type='string',
-        help='rrset or rdata rrtype')
-    parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string',
-        help='rrset bailiwick')
+    parser.add_option('-c', '--config', dest='config', help='config file', action='append')
+    parser.add_option('-r', '--rrset', dest='rrset', type='string', help='rrset <ONAME>[/<RRTYPE>[/BAILIWICK]]')
+    parser.add_option('-n', '--rdataname', dest='rdata_name', type='string', help='rdata name <NAME>[/<RRTYPE>]')
+    parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string', help='rdata ip <IPADDRESS|IPRANGE|IPNETWORK>')
+    parser.add_option('-t', '--rrtype', dest='rrtype', type='string', help='rrset or rdata rrtype')
+    parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string', help='rrset bailiwick')
     parser.add_option('-s', '--sort', dest='sort', type='string', help='sort key')
-    parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False,
-        help='reverse sort')
-    parser.add_option('-j', '--json', dest='json', action='store_true', default=False,
-        help='output in JSON format')
-    parser.add_option('-l', '--limit', dest='limit', type='int', default=0,
-        help='limit number of results')
+    parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False, help='reverse sort')
+    parser.add_option('-j', '--json', dest='json', action='store_true', default=False, help='output in JSON format')
+    parser.add_option('-l', '--limit', dest='limit', type='int', default=0, help='limit number of results')
 
     parser.add_option('', '--before', dest='before', type='string', help='only output results seen before this time')
     parser.add_option('', '--after', dest='after', type='string', help='only output results seen after this time')
@@ -263,20 +263,20 @@ def main():
         print(str(e), file=sys.stderr)
         sys.exit(1)
 
-    if not 'DNSDB_SERVER' in cfg:
+    if 'DNSDB_SERVER' not in cfg:
         cfg['DNSDB_SERVER'] = DEFAULT_DNSDB_SERVER
-    if not 'HTTP_PROXY' in cfg:
+    if 'HTTP_PROXY' not in cfg:
         cfg['HTTP_PROXY'] = DEFAULT_HTTP_PROXY
-    if not 'HTTPS_PROXY' in cfg:
+    if 'HTTPS_PROXY' not in cfg:
         cfg['HTTPS_PROXY'] = DEFAULT_HTTPS_PROXY
-    if not 'APIKEY' in cfg:
+    if 'APIKEY' not in cfg:
         sys.stderr.write('dnsdb_query: APIKEY not defined in config file\n')
         sys.exit(1)
 
     client = DnsdbClient(cfg['DNSDB_SERVER'], cfg['APIKEY'],
-            limit=options.limit,
-            http_proxy=cfg['HTTP_PROXY'],
-            https_proxy=cfg['HTTPS_PROXY'])
+                         limit=options.limit,
+                         http_proxy=cfg['HTTP_PROXY'],
+                         https_proxy=cfg['HTTPS_PROXY'])
     if options.rrset:
         if options.rrtype or options.bailiwick:
             qargs = (options.rrset, options.rrtype, options.bailiwick)
@@ -307,7 +307,7 @@ def main():
         if options.sort:
             results = list(results)
             if len(results) > 0:
-                if not options.sort in results[0]:
+                if options.sort not in results[0]:
                     sort_keys = results[0].keys()
                     sort_keys.sort()
                     sys.stderr.write('dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' % (options.sort, ', '.join(sort_keys)))
@@ -319,5 +319,6 @@ def main():
         print(e.message, file=sys.stderr)
         sys.exit(1)
 
+
 if __name__ == '__main__':
     main()

misp_modules/modules/expansion/asn_history.py

@@ -22,9 +22,9 @@ def handler(q=False):
         misperrors['error'] = "Unsupported attributes type"
         return misperrors
 
-    if not request.get('config') and not (request['config'].get('host') and
-                                          request['config'].get('port') and
-                                          request['config'].get('db')):
+    if not request.get('config') and not (request['config'].get('host')
+                                          and request['config'].get('port')
+                                          and request['config'].get('db')):
         misperrors['error'] = 'ASN description history configuration is missing'
         return misperrors
 

misp_modules/modules/expansion/btc_steroids.py

@@ -12,24 +12,25 @@ moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
 
 moduleconfig = []
 
-blockchain_firstseen='https://blockchain.info/q/addressfirstseen/'
-blockchain_balance='https://blockchain.info/q/addressbalance/'
-blockchain_totalreceived='https://blockchain.info/q/getreceivedbyaddress/'
-blockchain_all='https://blockchain.info/rawaddr/'
-converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts='
+blockchain_firstseen = 'https://blockchain.info/q/addressfirstseen/'
+blockchain_balance = 'https://blockchain.info/q/addressbalance/'
+blockchain_totalreceived = 'https://blockchain.info/q/getreceivedbyaddress/'
+blockchain_all = 'https://blockchain.info/rawaddr/{}?filter=5{}'
+converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts={}'
 converter_rls = 'https://min-api.cryptocompare.com/stats/rate/limit'
 result_text = ""
 g_rate_limit = 300
 start_time = 0
 conversion_rates = {}
 
+
 def get_consumption(output=False):
     try:
         req = requests.get(converter_rls)
         jreq = req.json()
         minute = str(jreq['Data']['calls_left']['minute'])
-        hour   = str(jreq['Data']['calls_left']['hour'])
-    except:
+        hour = str(jreq['Data']['calls_left']['hour'])
+    except Exception:
         minute = str(-1)
         hour = str(-1)
     # Debug out for the console
@@ -53,20 +54,20 @@ def convert(btc, timestamp):
             minute, hour = get_consumption()
         g_rate_limit -= 1
         now = time.time()
-        delta = now - start_time
-        #print(g_rate_limit)
+        # delta = now - start_time
+        # print(g_rate_limit)
         if g_rate_limit <= 10:
             minute, hour = get_consumption(output=True)
             if int(minute) <= 10:
-                #print(minute)
-                #get_consumption(output=True)
+                # print(minute)
+                # get_consumption(output=True)
                 time.sleep(3)
             else:
                 mprint(minute)
                 start_time = time.time()
                 g_rate_limit = int(minute)
         try:
-            req = requests.get(converter+str(timestamp))
+            req = requests.get(converter.format(timestamp))
             jreq = req.json()
             usd = jreq['BTC']['USD']
             eur = jreq['BTC']['EUR']
@@ -78,7 +79,7 @@ def convert(btc, timestamp):
     # Actually convert and return the values
     u = usd * btc
     e = eur * btc
-    return u,e
+    return u, e
 
 
 def mprint(input):
@@ -90,8 +91,8 @@ def mprint(input):
 def handler(q=False):
     global result_text
     global conversion_rates
-    start_time = time.time()
-    now = time.time()
+    # start_time = time.time()
+    # now = time.time()
     if q is False:
         return False
     request = json.loads(q)
@@ -107,13 +108,13 @@ def handler(q=False):
 
     mprint("\nAddress:\t" + btc)
     try:
-        req = requests.get(blockchain_all+btc+"?limit=50&filter=5")
+        req = requests.get(blockchain_all.format(btc, "&limit=50"))
         jreq = req.json()
-    except Exception as e:
-        #print(e)
+    except Exception:
+        # print(e)
         print(req.text)
         result_text = ""
-        sys.exit(1) 
+        sys.exit(1)
 
     n_tx = jreq['n_tx']
     balance = float(jreq['final_balance'] / 100000000)
@@ -130,11 +131,11 @@ def handler(q=False):
     i = 0
     while i < n_tx:
         if click is False:
-            req = requests.get(blockchain_all+btc+"?limit=5&offset="+str(i)+"&filter=5")
+            req = requests.get(blockchain_all.format(btc, "&limit=5&offset={}".format(i)))
             if n_tx > 5:
                 n_tx = 5
         else:
-            req = requests.get(blockchain_all+btc+"?limit=50&offset="+str(i)+"&filter=5")
+            req = requests.get(blockchain_all.format(btc, "&limit=50&offset={}".format(i)))
         jreq = req.json()
         if jreq['txs']:
             for transactions in jreq['txs']:
@@ -144,8 +145,8 @@ def handler(q=False):
                     script_old = tx['script']
                     if tx['prev_out']['value'] != 0 and tx['prev_out']['addr'] == btc:
                         datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time'])))
-                        value = float(tx['prev_out']['value'] / 100000000 )
-                        u,e = convert(value, transactions['time'])
+                        value = float(tx['prev_out']['value'] / 100000000)
+                        u, e = convert(value, transactions['time'])
                         mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0'))
                         if script_old != tx['script']:
                             i += 1
@@ -153,19 +154,19 @@ def handler(q=False):
                             sum_counter += 1
                             sum += value
                 if sum_counter > 1:
-                    u,e = convert(sum, transactions['time'])
+                    u, e = convert(sum, transactions['time'])
                     mprint("\t\t\t\t\t----------------------------------------------")
                     mprint("#" + str(n_tx - i) + "\t\t\t\t  Sum:\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR\n".format(sum, u, e).rstrip('0'))
                 for tx in transactions['out']:
                     if tx['value'] != 0 and tx['addr'] == btc:
                         datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time'])))
-                        value = float(tx['value'] / 100000000 )
-                        u,e = convert(value, transactions['time'])
+                        value = float(tx['value'] / 100000000)
+                        u, e = convert(value, transactions['time'])
                         mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t {0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0'))
-                        #i += 1
+                        # i += 1
                 i += 1
 
-    r = { 
+    r = {
         'results': [
             {
                 'types': ['text'],

misp_modules/modules/expansion/countrycode.py

@@ -12,20 +12,21 @@ moduleinfo = {'version': '1', 'author': 'Hannah Ward',
 # config fields that your code expects from the site admin
 moduleconfig = []
 
-common_tlds = {"com":"Commercial (Worldwide)",
-               "org":"Organisation (Worldwide)",
-               "net":"Network (Worldwide)",
-               "int":"International (Worldwide)",
-               "edu":"Education (Usually USA)",
-               "gov":"Government (USA)"
-              }
+common_tlds = {"com": "Commercial (Worldwide)",
+               "org": "Organisation (Worldwide)",
+               "net": "Network (Worldwide)",
+               "int": "International (Worldwide)",
+               "edu": "Education (Usually USA)",
+               "gov": "Government (USA)"
+               }
 
 codes = False
 
+
 def handler(q=False):
     global codes
     if not codes:
-      codes = requests.get("http://www.geognos.com/api/en/countries/info/all.json").json()
+        codes = requests.get("http://www.geognos.com/api/en/countries/info/all.json").json()
     if q is False:
         return False
     request = json.loads(q)
@@ -36,18 +37,18 @@ def handler(q=False):
 
     # Check if it's a common, non country one
     if ext in common_tlds.keys():
-      val = common_tlds[ext]
+        val = common_tlds[ext]
     else:
-      # Retrieve a json full of country info
-      if not codes["StatusMsg"] == "OK":
-        val = "Unknown"
-      else:
-        # Find our code based on TLD
-        codes = codes["Results"]
-        for code in codes.keys():
-          if codes[code]["CountryCodes"]["tld"] == ext:
-            val = codes[code]["Name"]
-    r = {'results': [{'types':['text'], 'values':[val]}]}
+        # Retrieve a json full of country info
+        if not codes["StatusMsg"] == "OK":
+            val = "Unknown"
+        else:
+            # Find our code based on TLD
+            codes = codes["Results"]
+            for code in codes.keys():
+                if codes[code]["CountryCodes"]["tld"] == ext:
+                    val = codes[code]["Name"]
+    r = {'results': [{'types': ['text'], 'values':[val]}]}
     return r
 
 
@@ -58,4 +59,3 @@ def introspection():
 def version():
     moduleinfo['config'] = moduleconfig
     return moduleinfo
-

misp_modules/modules/expansion/dbl_spamhaus.py

@@ -1,6 +1,5 @@
 import json
-import datetime
-from collections import defaultdict
+import sys
 
 try:
     import dns.resolver
@@ -30,12 +29,14 @@ dbl_mapping = {'127.0.1.2': 'spam domain',
                '127.0.1.106': 'abused legit botnet C&C',
                '127.0.1.255': 'IP queries prohibited!'}
 
+
 def fetch_requested_value(request):
     for attribute_type in mispattributes['input']:
         if request.get(attribute_type):
             return request[attribute_type].split('|')[0]
     return None
 
+
 def handler(q=False):
     if q is False:
         return False
@@ -52,9 +53,11 @@ def handler(q=False):
         result = str(e)
     return {'results': [{'types': mispattributes.get('output'), 'values': result}]}
 
+
 def introspection():
     return mispattributes
 
+
 def version():
     moduleinfo['config'] = moduleconfig
     return moduleinfo

misp_modules/modules/expansion/dns.py

@@ -43,7 +43,7 @@ def handler(q=False):
     except dns.exception.Timeout:
         misperrors['error'] = "Timeout"
         return misperrors
-    except:
+    except Exception:
         misperrors['error'] = "DNS resolving error"
         return misperrors
 

misp_modules/modules/expansion/farsight_passivedns.py

@@ -51,7 +51,7 @@ def lookup_name(client, name):
                 for i in item.get('rdata'):
                     # grab email field and replace first dot by @ to convert to an email address
                     yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1))
-    except QueryError as e:
+    except QueryError:
         pass
 
     try:
@@ -59,7 +59,7 @@ def lookup_name(client, name):
         for item in res:
             if item.get('rrtype') in ['A', 'AAAA', 'CNAME']:
                 yield(item.get('rrname').rstrip('.'))
-    except QueryError as e:
+    except QueryError:
         pass
 
 
@@ -68,7 +68,7 @@ def lookup_ip(client, ip):
         res = client.query_rdata_ip(ip)
         for item in res:
             yield(item['rrname'].rstrip('.'))
-    except QueryError as e:
+    except QueryError:
         pass
 
 

misp_modules/modules/expansion/geoip_country.py

@@ -27,7 +27,7 @@ try:
     config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'geoip_country.cfg'))
     gi = pygeoip.GeoIP(config.get('GEOIP', 'database'))
     enabled = True
-except:
+except Exception:
     enabled = False
 
 
@@ -49,7 +49,7 @@ def handler(q=False):
 
     try:
         answer = gi.country_code_by_addr(toquery)
-    except:
+    except Exception:
         misperrors['error'] = "GeoIP resolving error"
         return misperrors
 

misp_modules/modules/expansion/hashdd.py

@@ -20,7 +20,7 @@ def handler(q=False):
     if v is None:
         misperrors['error'] = 'Hash value is missing.'
         return misperrors
-    r = requests.post(hashddapi_url, data={'hash':v})
+    r = requests.post(hashddapi_url, data={'hash': v})
     if r.status_code == 200:
         state = json.loads(r.text)
         if state:

misp_modules/modules/expansion/ipasn.py

@@ -24,9 +24,9 @@ def handler(q=False):
         misperrors['error'] = "Unsupported attributes type"
         return misperrors
 
-    if not request.get('config') and not (request['config'].get('host') and
-                                          request['config'].get('port') and
-                                          request['config'].get('db')):
+    if not request.get('config') and not (request['config'].get('host')
+                                          and request['config'].get('port')
+                                          and request['config'].get('db')):
         misperrors['error'] = 'IP ASN history configuration is missing'
         return misperrors
 

misp_modules/modules/expansion/iprep.py

@@ -45,7 +45,7 @@ def parse_iprep(ip, api):
     url = 'https://www.packetmail.net/iprep.php/%s' % ip
     try:
         data = requests.get(url, params={'apikey': api}).json()
-    except:
+    except Exception:
         return ['Error pulling data'], rep
     # print '%s' % data
     for name, val in data.items():
@@ -71,11 +71,11 @@ def parse_iprep(ip, api):
                 misp_val = context
                 full_text += '\n%s' % context
                 misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen'])
-                rep.append({'types': mispattributes['output'], 'categories':['External analysis'], 'values': misp_val, 'comment': misp_comment})
-            except:
+                rep.append({'types': mispattributes['output'], 'categories': ['External analysis'], 'values': misp_val, 'comment': misp_comment})
+            except Exception:
                 err.append('Error parsing source: %s' % name)
 
-    rep.append({'types': ['freetext'], 'values': full_text , 'comment': 'Free text import of IPRep'})
+    rep.append({'types': ['freetext'], 'values': full_text, 'comment': 'Free text import of IPRep'})
     return err, rep
 
 

misp_modules/modules/expansion/macaddress_io.py

@@ -86,32 +86,30 @@ def handler(q=False):
         response.block_details.date_updated.strftime('%d %B %Y') if response.block_details.date_updated else None
 
     results = {
-        'results': [
-            {'types': ['text'], 'values':
-                {
-                    # Mac address details
-                    'Valid MAC address': "True" if response.mac_address_details.is_valid else "False",
-                    'Transmission type': response.mac_address_details.transmission_type,
-                    'Administration type': response.mac_address_details.administration_type,
-
-                    # Vendor details
-                    'OUI': response.vendor_details.oui,
-                    'Vendor details are hidden': "True" if response.vendor_details.is_private else "False",
-                    'Company name': response.vendor_details.company_name,
-                    'Company\'s address': response.vendor_details.company_address,
-                    'County code': response.vendor_details.country_code,
-
-                    # Block details
-                    'Block found': "True" if response.block_details.block_found else "False",
-                    'The left border of the range': response.block_details.border_left,
-                    'The right border of the range': response.block_details.border_right,
-                    'The total number of MAC addresses in this range': response.block_details.block_size,
-                    'Assignment block size': response.block_details.assignment_block_size,
-                    'Date when the range was allocated': date_created,
-                    'Date when the range was last updated': date_updated
-                }
-             }
-        ]
+        'results':
+        [{'types': ['text'], 'values':
+            {
+                # Mac address details
+                'Valid MAC address': "True" if response.mac_address_details.is_valid else "False",
+                'Transmission type': response.mac_address_details.transmission_type,
+                'Administration type': response.mac_address_details.administration_type,
+
+                # Vendor details
+                'OUI': response.vendor_details.oui,
+                'Vendor details are hidden': "True" if response.vendor_details.is_private else "False",
+                'Company name': response.vendor_details.company_name,
+                'Company\'s address': response.vendor_details.company_address,
+                'County code': response.vendor_details.country_code,
+
+                # Block details
+                'Block found': "True" if response.block_details.block_found else "False",
+                'The left border of the range': response.block_details.border_left,
+                'The right border of the range': response.block_details.border_right,
+                'The total number of MAC addresses in this range': response.block_details.block_size,
+                'Assignment block size': response.block_details.assignment_block_size,
+                'Date when the range was allocated': date_created,
+                'Date when the range was last updated': date_updated
+        }}]
     }
 
     return results

misp_modules/modules/expansion/onyphe.py

@@ -1,4 +1,3 @@
-import json
 # -*- coding: utf-8 -*-
 
 import json
@@ -9,7 +8,8 @@ except ImportError:
 
 misperrors = {'error': 'Error'}
 
-mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], 'output': ['hostname', 'domain', 'ip-src', 'ip-dst','url']}
+mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
+                  'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']}
 # possible module-types: 'expansion', 'hover' or both
 moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
               'description': 'Query on Onyphe',
@@ -54,7 +54,7 @@ def handle_expansion(api, ip, misperrors):
         misperrors['error'] = result['message']
         return misperrors
 
-    categories = list(set([item['@category'] for item in result['results']]))
+    # categories = list(set([item['@category'] for item in result['results']]))
 
     result_filtered = {"results": []}
     urls_pasties = []
@@ -72,9 +72,9 @@ def handle_expansion(api, ip, misperrors):
             os_target = r['os']
             if os_target != 'Unknown':
                 os_list.append(r['os'])
-        elif r['@category'] == 'resolver' and r['type'] =='reverse':
+        elif r['@category'] == 'resolver' and r['type'] == 'reverse':
             domains_resolver.append(r['reverse'])
-        elif r['@category'] == 'resolver' and r['type'] =='forward':
+        elif r['@category'] == 'resolver' and r['type'] == 'forward':
             domains_forward.append(r['forward'])
 
     result_filtered['results'].append({'types': ['url'], 'values': urls_pasties,
@@ -90,7 +90,7 @@ def handle_expansion(api, ip, misperrors):
     result_filtered['results'].append({'types': ['domain'],
                                        'values': list(set(domains_resolver)),
                                        'categories': ['Network activity'],
-                                       'comment': 'resolver to %s' % ip })
+                                       'comment': 'resolver to %s' % ip})
 
     result_filtered['results'].append({'types': ['domain'],
                                        'values': list(set(domains_forward)),

misp_modules/modules/expansion/onyphe_full.py

@@ -1,4 +1,3 @@
-import json
 # -*- coding: utf-8 -*-
 
 import json
@@ -10,7 +9,7 @@ except ImportError:
 misperrors = {'error': 'Error'}
 
 mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
-                  'output': ['hostname', 'domain', 'ip-src', 'ip-dst','url']}
+                  'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']}
 
 # possible module-types: 'expansion', 'hover' or both
 moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
@@ -38,10 +37,10 @@ def handler(q=False):
         ip = ''
         if request.get('ip-src'):
             ip = request['ip-src']
-            return handle_ip(api ,ip, misperrors)
+            return handle_ip(api, ip, misperrors)
         elif request.get('ip-dst'):
             ip = request['ip-dst']
-            return handle_ip(api,ip,misperrors)
+            return handle_ip(api, ip, misperrors)
         elif request.get('domain'):
             domain = request['domain']
             return handle_domain(api, domain, misperrors)
@@ -91,11 +90,11 @@ def handle_ip(api, ip, misperrors):
     r, status_ok = expand_syscan(api, ip, misperrors)
 
     if status_ok:
-         result_filtered['results'].extend(r)
+        result_filtered['results'].extend(r)
     else:
-         misperrors['error'] = "Error syscan result"
+        misperrors['error'] = "Error syscan result"
 
-    r, status_ok = expand_pastries(api,misperrors,ip=ip)
+    r, status_ok = expand_pastries(api, misperrors, ip=ip)
 
     if status_ok:
         result_filtered['results'].extend(r)
@@ -185,11 +184,11 @@ def expand_syscan(api, ip, misperror):
     return r, status_ok
 
 
-def expand_datascan(api, misperror,**kwargs):
+def expand_datascan(api, misperror, **kwargs):
     status_ok = False
     r = []
-    ip = ''
-    query =''
+    # ip = ''
+    query = ''
     asn_list = []
     geoloc = []
     orgs = []
@@ -311,7 +310,7 @@ def expand_pastries(api, misperror, **kwargs):
         query = kwargs.get('domain')
         result = api.search_pastries('domain:%s' % query)
 
-    if result['status'] =='ok':
+    if result['status'] == 'ok':
         status_ok = True
         for item in result['results']:
             if item['@category'] == 'pastries':
@@ -328,7 +327,7 @@ def expand_pastries(api, misperror, **kwargs):
         r.append({'types': ['url'],
                   'values': urls_pasties,
                   'categories': ['External analysis'],
-                  'comment':'URLs of pasties where %s has found' % query})
+                  'comment': 'URLs of pasties where %s has found' % query})
         r.append({'types': ['domain'], 'values': list(set(domains)),
                   'categories': ['Network activity'],
                   'comment': 'Domains found in pasties of Onyphe'})
@@ -340,7 +339,7 @@ def expand_pastries(api, misperror, **kwargs):
     return r, status_ok
 
 
-def expand_threatlist(api, misperror,**kwargs):
+def expand_threatlist(api, misperror, **kwargs):
     status_ok = False
     r = []
 
@@ -366,7 +365,8 @@ def expand_threatlist(api, misperror,**kwargs):
                   'comment': '%s is present in threatlist' % query
                   })
 
-    return r,status_ok
+    return r, status_ok
+
 
 def introspection():
     return mispattributes

misp_modules/modules/expansion/otx.py

@@ -15,9 +15,10 @@ moduleinfo = {'version': '1', 'author': 'chrisdoman',
 # We're not actually using the API key yet
 moduleconfig = ["apikey"]
 
+
 # Avoid adding windows update to enrichment etc.
 def isBlacklisted(value):
-    blacklist = ['0.0.0.0', '8.8.8.8', '255.255.255.255', '192.168.56.' , 'time.windows.com']
+    blacklist = ['0.0.0.0', '8.8.8.8', '255.255.255.255', '192.168.56.', 'time.windows.com']
 
     for b in blacklist:
         if value in b:
@@ -25,10 +26,12 @@ def isBlacklisted(value):
 
     return True
 
+
 def valid_ip(ip):
     m = re.match(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$", ip)
     return bool(m) and all(map(lambda n: 0 <= int(n) <= 255, m.groups()))
 
+
 def findAll(data, keys):
     a = []
     if isinstance(data, dict):
@@ -43,9 +46,11 @@ def findAll(data, keys):
             a.extend(findAll(i, keys))
     return a
 
+
 def valid_email(email):
     return bool(re.search(r"[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?", email))
 
+
 def handler(q=False):
     if q is False:
         return False
@@ -99,19 +104,17 @@ def getHash(_hash, key):
 
 def getIP(ip, key):
     ret = []
-    req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/malware/" + ip + "?limit=1000").text )
+    req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/ip/malware/" + ip + "?limit=1000").text)
 
     for _hash in findAll(req, "hash"):
         ret.append({"types": ["sha256"], "values": [_hash]})
 
-
-    req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text )
+    req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text)
 
     for hostname in findAll(req, "hostname"):
         if not isBlacklisted(hostname):
             ret.append({"types": ["hostname"], "values": [hostname]})
 
-
     return ret
 
 
@@ -119,7 +122,7 @@ def getDomain(domain, key):
 
     ret = []
 
-    req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text )
+    req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text)
 
     for _hash in findAll(req, "hash"):
         ret.append({"types": ["sha256"], "values": [_hash]})
@@ -144,6 +147,7 @@ def getDomain(domain, key):
 
     return ret
 
+
 def introspection():
     return mispattributes
 

misp_modules/modules/expansion/passivetotal.py

@@ -331,7 +331,7 @@ def handler(q=False):
             output['results'] += results
         else:
             log.error("Unsupported query pattern issued.")
-    except:
+    except Exception:
         return misperrors
 
     return output

misp_modules/modules/expansion/rbl.py

@@ -1,5 +1,5 @@
 import json
-import datetime
+import sys
 
 try:
     import dns.resolver
@@ -18,64 +18,65 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
 moduleconfig = []
 
 rbls = {
- 'spam.spamrats.com': 'http://www.spamrats.com',
- 'spamguard.leadmon.net': 'http://www.leadmon.net/SpamGuard/',
- 'rbl-plus.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
- 'web.dnsbl.sorbs.net': 'http://www.sorbs.net',
- 'ix.dnsbl.manitu.net': 'http://www.dnsbl.manitu.net',
- 'virus.rbl.jp': 'http://www.rbl.jp',
- 'dul.dnsbl.sorbs.net': 'http://www.sorbs.net',
- 'bogons.cymru.com': 'http://www.team-cymru.org/Services/Bogons/',
- 'psbl.surriel.com': 'http://psbl.surriel.com',
- 'misc.dnsbl.sorbs.net': 'http://www.sorbs.net',
- 'httpbl.abuse.ch': 'http://dnsbl.abuse.ch',
- 'combined.njabl.org': 'http://combined.njabl.org',
- 'smtp.dnsbl.sorbs.net': 'http://www.sorbs.net',
- 'korea.services.net': 'http://korea.services.net',
- 'drone.abuse.ch': 'http://dnsbl.abuse.ch',
- 'rbl.efnetrbl.org': 'http://rbl.efnetrbl.org',
- 'cbl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US',
- 'b.barracudacentral.org': 'http://www.barracudacentral.org/rbl/removal-request',
- 'bl.spamcannibal.org': 'http://www.spamcannibal.org',
- 'xbl.spamhaus.org': 'http://www.spamhaus.org/xbl/',
- 'zen.spamhaus.org': 'http://www.spamhaus.org/zen/',
- 'rbl.suresupport.com': 'http://suresupport.com/postmaster',
- 'db.wpbl.info': 'http://www.wpbl.info',
- 'sbl.spamhaus.org': 'http://www.spamhaus.org/sbl/',
- 'http.dnsbl.sorbs.net': 'http://www.sorbs.net',
- 'csi.cloudmark.com': 'http://www.cloudmark.com/en/products/cloudmark-sender-intelligence/index',
- 'rbl.interserver.net': 'http://rbl.interserver.net',
- 'ubl.unsubscore.com': 'http://www.lashback.com/blacklist/',
- 'dnsbl.sorbs.net': 'http://www.sorbs.net',
- 'virbl.bit.nl': 'http://virbl.bit.nl',
- 'pbl.spamhaus.org': 'http://www.spamhaus.org/pbl/',
- 'socks.dnsbl.sorbs.net': 'http://www.sorbs.net',
- 'short.rbl.jp': 'http://www.rbl.jp',
- 'dnsbl.dronebl.org': 'http://www.dronebl.org',
- 'blackholes.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
- 'truncate.gbudb.net': 'http://www.gbudb.com/truncate/index.jsp',
- 'dyna.spamrats.com': 'http://www.spamrats.com',
- 'spamrbl.imp.ch': 'http://antispam.imp.ch',
- 'spam.dnsbl.sorbs.net': 'http://www.sorbs.net',
- 'wormrbl.imp.ch': 'http://antispam.imp.ch',
- 'query.senderbase.org': 'http://www.senderbase.org/about',
- 'opm.tornevall.org': 'http://dnsbl.tornevall.org',
- 'netblock.pedantic.org': 'http://pedantic.org',
- 'access.redhawk.org': 'http://www.redhawk.org/index.php?option=com_wrapper&Itemid=33',
- 'cdl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US',
- 'multi.surbl.org': 'http://www.surbl.org',
- 'noptr.spamrats.com': 'http://www.spamrats.com',
- 'dnsbl.inps.de': 'http://dnsbl.inps.de/index.cgi?lang=en',
- 'bl.spamcop.net': 'http://bl.spamcop.net',
- 'cbl.abuseat.org': 'http://cbl.abuseat.org',
- 'dsn.rfc-ignorant.org': 'http://www.rfc-ignorant.org/policy-dsn.php',
- 'zombie.dnsbl.sorbs.net': 'http://www.sorbs.net',
- 'dnsbl.njabl.org': 'http://dnsbl.njabl.org',
- 'relays.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
- 'rbl.spamlab.com': 'http://tools.appriver.com/index.aspx?tool=rbl',
- 'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html'
+    'spam.spamrats.com': 'http://www.spamrats.com',
+    'spamguard.leadmon.net': 'http://www.leadmon.net/SpamGuard/',
+    'rbl-plus.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
+    'web.dnsbl.sorbs.net': 'http://www.sorbs.net',
+    'ix.dnsbl.manitu.net': 'http://www.dnsbl.manitu.net',
+    'virus.rbl.jp': 'http://www.rbl.jp',
+    'dul.dnsbl.sorbs.net': 'http://www.sorbs.net',
+    'bogons.cymru.com': 'http://www.team-cymru.org/Services/Bogons/',
+    'psbl.surriel.com': 'http://psbl.surriel.com',
+    'misc.dnsbl.sorbs.net': 'http://www.sorbs.net',
+    'httpbl.abuse.ch': 'http://dnsbl.abuse.ch',
+    'combined.njabl.org': 'http://combined.njabl.org',
+    'smtp.dnsbl.sorbs.net': 'http://www.sorbs.net',
+    'korea.services.net': 'http://korea.services.net',
+    'drone.abuse.ch': 'http://dnsbl.abuse.ch',
+    'rbl.efnetrbl.org': 'http://rbl.efnetrbl.org',
+    'cbl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US',
+    'b.barracudacentral.org': 'http://www.barracudacentral.org/rbl/removal-request',
+    'bl.spamcannibal.org': 'http://www.spamcannibal.org',
+    'xbl.spamhaus.org': 'http://www.spamhaus.org/xbl/',
+    'zen.spamhaus.org': 'http://www.spamhaus.org/zen/',
+    'rbl.suresupport.com': 'http://suresupport.com/postmaster',
+    'db.wpbl.info': 'http://www.wpbl.info',
+    'sbl.spamhaus.org': 'http://www.spamhaus.org/sbl/',
+    'http.dnsbl.sorbs.net': 'http://www.sorbs.net',
+    'csi.cloudmark.com': 'http://www.cloudmark.com/en/products/cloudmark-sender-intelligence/index',
+    'rbl.interserver.net': 'http://rbl.interserver.net',
+    'ubl.unsubscore.com': 'http://www.lashback.com/blacklist/',
+    'dnsbl.sorbs.net': 'http://www.sorbs.net',
+    'virbl.bit.nl': 'http://virbl.bit.nl',
+    'pbl.spamhaus.org': 'http://www.spamhaus.org/pbl/',
+    'socks.dnsbl.sorbs.net': 'http://www.sorbs.net',
+    'short.rbl.jp': 'http://www.rbl.jp',
+    'dnsbl.dronebl.org': 'http://www.dronebl.org',
+    'blackholes.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
+    'truncate.gbudb.net': 'http://www.gbudb.com/truncate/index.jsp',
+    'dyna.spamrats.com': 'http://www.spamrats.com',
+    'spamrbl.imp.ch': 'http://antispam.imp.ch',
+    'spam.dnsbl.sorbs.net': 'http://www.sorbs.net',
+    'wormrbl.imp.ch': 'http://antispam.imp.ch',
+    'query.senderbase.org': 'http://www.senderbase.org/about',
+    'opm.tornevall.org': 'http://dnsbl.tornevall.org',
+    'netblock.pedantic.org': 'http://pedantic.org',
+    'access.redhawk.org': 'http://www.redhawk.org/index.php?option=com_wrapper&Itemid=33',
+    'cdl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US',
+    'multi.surbl.org': 'http://www.surbl.org',
+    'noptr.spamrats.com': 'http://www.spamrats.com',
+    'dnsbl.inps.de': 'http://dnsbl.inps.de/index.cgi?lang=en',
+    'bl.spamcop.net': 'http://bl.spamcop.net',
+    'cbl.abuseat.org': 'http://cbl.abuseat.org',
+    'dsn.rfc-ignorant.org': 'http://www.rfc-ignorant.org/policy-dsn.php',
+    'zombie.dnsbl.sorbs.net': 'http://www.sorbs.net',
+    'dnsbl.njabl.org': 'http://dnsbl.njabl.org',
+    'relays.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
+    'rbl.spamlab.com': 'http://tools.appriver.com/index.aspx?tool=rbl',
+    'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html'
 }
 
+
 def handler(q=False):
     if q is False:
         return False
@@ -89,11 +90,11 @@ def handler(q=False):
         return misperrors
     listed = []
     info = []
-    ipRev =  '.'.join(ip.split('.')[::-1])
+    ipRev = '.'.join(ip.split('.')[::-1])
     for rbl in rbls:
         query = '{}.{}'.format(ipRev, rbl)
         try:
-            txt = resolver.query(query,'TXT')
+            txt = resolver.query(query, 'TXT')
             listed.append(query)
             info.append([str(t) for t in txt])
         except Exception: