Merge branch 'master' into utf_hate

.travis.yml

@@ -9,12 +9,12 @@ python:
     - "3.4"
     - "3.5"
     - "3.5-dev"
+    - "3.6"
+    - "3.6-dev"
     - "nightly"
 
 install:
-    - pip install -U nose
+    - pip install -U nose codecov pytest
-    - pip install coveralls
-    - pip install codecov
     - pip install -U -r REQUIREMENTS
     - pip install .
 
@@ -35,4 +35,3 @@ script:
 after_success:
     - coverage combine .coverage*
     - codecov
-    - coveralls

README.md

@@ -26,19 +26,26 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
 * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
 * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
+* [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
 * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
+* [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module.
 * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
+* [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
 * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
 * [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key)
+* [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module.
+* [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module.
 
 ### Export modules
 
 * [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF).
+* [Lite Export](/misp-modules/blob/master/misp_modules/modules/export_mod/liteexport.py) module to export a lite event.
 
 ### Import modules
 
 * [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import.
 * [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes.
+* [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library.
 * [stiximport](misp_modules/modules/import_mod/stiximport.py) - An import module to process STIX xml/json.
 * [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata.
 * [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export.
@@ -53,7 +60,7 @@ cd misp-modules
 sudo pip3 install -I -r REQUIREMENTS
 sudo pip3 install -I .
 sudo vi /etc/rc.local, add this line: `sudo -u www-data misp-modules -s &`
-/usr/local/bin/misp-modules #to start the modules
+misp-modules #to start the modules
 ~~~~
 
 ## How to add your own MISP modules?
@@ -352,6 +359,23 @@ Recommended     Plugin.Import_ocr_enabled       true   Enable or disable the ocr
 
 In this same menu set any other plugin settings that are required for testing.
 
+## Install misp-module on an offline instance.
+First, you need to grab all necessery packages for example like this : 
+
+Use pip wheel to create an archive
+~~~
+mkdir misp-modules-offline
+pip3 wheel -r REQUIREMENTS shodan --wheel-dir=./misp-modules-offline
+tar -cjvf misp-module-bundeled.tar.bz2 ./misp-modules-offline/*
+~~~
+On offline machine : 
+~~~
+mkdir misp-modules-bundle
+tar xvf misp-module-bundeled.tar.bz2 -C misp-modules-bundle
+cd misp-modules-bundle
+ls -1|while read line; do sudo pip3 install --force-reinstall --ignore-installed --upgrade --no-index --no-deps ${line};done
+~~~
+Next you can follow standard install procedure.
 
 ## How to contribute your own module?
 

REQUIREMENTS

@@ -13,9 +13,10 @@ ipasn-redis
 asnhistory
 git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client
 git+https://github.com/MISP/MISP-STIX-Converter.git#egg=misp_stix_converter
-git+https://github.com/CIRCL/PyMISP.git#egg=pymisp
+git+https://github.com/MISP/PyMISP.git#egg=pymisp
 pillow
 pytesseract
 SPARQLWrapper
 domaintools_api
 pygeoip
+bs4

misp_modules/modules/expansion/__init__.py

@@ -2,4 +2,4 @@ from . import _vmray
 
 __all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl',
            'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'ipasn', 'passivetotal', 'sourcecache',
-           'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki']
+           'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer']

misp_modules/modules/expansion/iprep.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+
+import json
+import requests
+
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['text']}
+moduleinfo = {'version': '1.0', 'author': 'Keith Faber',
+              'description': 'Query IPRep Data for IP Address',
+              'module-type': ['expansion']}
+
+moduleconfig = ['apikey']
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if request.get('ip-src'):
+        toquery = request['ip-src']
+    elif request.get('ip-dst'):
+        toquery = request['ip-dst']
+    else:
+        misperrors['error'] = "Unsupported attributes type"
+        return misperrors
+
+    if not request.get('config') and not request['config'].get('apikey'):
+        misperrors['error'] = 'IPRep api key is missing'
+        return misperrors
+
+    err, rep = parse_iprep(toquery, request['config'].get('apikey'))
+    if len(err) > 0:
+        misperrors['error'] = ','.join(err)
+        return misperrors
+    return {'results': rep}
+
+
+def parse_iprep(ip, api):
+    meta_fields = ['origin', 'Query_Time', 'created_on', 'IP_Lookup_History', 'IPs_in_collection', '_id', 'disclaimer',
+                   'MaxMind_Free_GeoIP', 'Unique_Lookups', 'query_result']
+    rep = []
+    err = []
+    full_text = ''
+    url = 'https://www.packetmail.net/iprep.php/%s' % ip
+    try:
+        data = requests.get(url, params={'apikey': api}).json()
+    except:
+        return ['Error pulling data'], rep
+    # print '%s' % data
+    for name, val in data.items():
+        if name not in meta_fields:
+            try:
+                context = val['context']
+                if type(context) is list:
+                    if context[0].get('alert'):
+                        context = ','.join([hit['alert']['signature'] for hit in context])
+                    elif context[0].get('signature'):
+                        context = ','.join([hit['signature'] for hit in context])
+                    elif context[0].get('target_port') and context[0].get('protocol'):
+                        context = ','.join(
+                            ['Port Attacked: %s %s' % (hit['target_port'], hit['protocol']) for hit in context])
+                    elif context[0].get('phishing_kit') and context[0].get('url'):
+                        context = ','.join(['%s (%s)' % (hit['phishing_kit'], hit['url']) for hit in context])
+                    else:
+                        context = ';'.join(['%s: %s' % (k, v) for k, v in context[0].items()])
+
+                if val.get('special_note'):
+                    context += '; ' + val['special_note']
+
+                misp_val = context
+                full_text += '\n%s' % context
+                misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen'])
+                rep.append({'types': mispattributes['output'], 'categories':['External analysis'], 'values': misp_val, 'comment': misp_comment})
+            except:
+                err.append('Error parsing source: %s' % name)
+
+    rep.append({'types': ['freetext'], 'values': full_text , 'comment': 'Free text import of IPRep'})
+    return err, rep
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/threatminer.py

@@ -0,0 +1,173 @@
+import json
+import requests
+from requests import HTTPError
+import base64
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'md5', 'sha1', 'sha256', 'sha512'],
+                  'output': ['domain', 'ip-src', 'ip-dst', 'text', 'md5', 'sha1', 'sha256', 'sha512', 'ssdeep',
+                             'authentihash', 'filename', 'whois-registrant-email', 'url', 'link']
+                  }
+
+# possible module-types: 'expansion', 'hover' or both
+moduleinfo = {'version': '1', 'author': 'KX499',
+              'description': 'Get information from ThreatMiner',
+              'module-type': ['expansion']}
+
+desc = '{}: Threatminer - {}'
+
+
+def handler(q=False):
+    if q is False:
+        return False
+
+    q = json.loads(q)
+
+    r = {'results': []}
+
+    if 'ip-src' in q:
+        r['results'] += get_ip(q['ip-src'])
+    if 'ip-dst' in q:
+        r['results'] += get_ip(q['ip-dst'])
+    if 'domain' in q:
+        r['results'] += get_domain(q['domain'])
+    if 'hostname' in q:
+        r['results'] += get_domain(q['hostname'])
+    if 'md5' in q:
+        r['results'] += get_hash(q['md5'])
+    if 'sha1' in q:
+        r['results'] += get_hash(q['sha1'])
+    if 'sha256' in q:
+        r['results'] += get_hash(q['sha256'])
+    if 'sha512' in q:
+        r['results'] += get_hash(q['sha512'])
+
+    uniq = []
+    for res in r['results']:
+        if res not in uniq:
+            uniq.append(res)
+    r['results'] = uniq
+    return r
+
+
+def get_domain(q):
+    ret = []
+    for flag in [1, 2, 3, 4, 5, 6]:
+        req = requests.get('https://www.threatminer.org/domain.php', params={'q': q, 'api': 'True', 'rt': flag})
+        if not req.status_code == 200:
+            continue
+        results = req.json().get('results')
+        if not results:
+            continue
+
+        for result in results:
+            if flag == 1: #whois
+                emails = result.get('whois', {}).get('emails')
+                if not emails:
+                    continue
+                for em_type, email in emails.items():
+                    ret.append({'types': ['whois-registrant-email'], 'values': [email], 'comment': desc.format(q, 'whois')})
+            if flag == 2: #pdns
+                ip = result.get('ip')
+                if ip:
+                    ret.append({'types': ['ip-src', 'ip-dst'], 'values': [ip], 'comment': desc.format(q, 'pdns')})
+            if flag == 3: #uri
+                uri = result.get('uri')
+                if uri:
+                    ret.append({'types': ['url'], 'values': [uri], 'comment': desc.format(q, 'uri')})
+            if flag == 4: #samples
+                if type(result) is str:
+                    ret.append({'types': ['sha256'], 'values': [result], 'comment': desc.format(q, 'samples')})
+            if flag == 5: #subdomains
+                if type(result) is str:
+                    ret.append({'types': ['domain'], 'values': [result], 'comment': desc.format(q, 'subdomain')})
+            if flag == 6: #reports
+                link = result.get('URL')
+                if link:
+                    ret.append({'types': ['url'], 'values': [link], 'comment': desc.format(q, 'report')})
+
+    return ret
+
+
+def get_ip(q):
+    ret = []
+    for flag in [1, 2, 3, 4, 5, 6]:
+        req = requests.get('https://www.threatminer.org/host.php', params={'q': q, 'api': 'True', 'rt': flag})
+        if not req.status_code == 200:
+            continue
+        results = req.json().get('results')
+        if not results:
+            continue
+
+        for result in results:
+            if flag == 1: #whois
+                emails = result.get('whois', {}).get('emails')
+                if not emails:
+                    continue
+                for em_type, email in emails.items():
+                    ret.append({'types': ['whois-registrant-email'], 'values': [email], 'comment': desc.format(q, 'whois')})
+            if flag == 2: #pdns
+                ip = result.get('ip')
+                if ip:
+                    ret.append({'types': ['ip-src', 'ip-dst'], 'values': [ip], 'comment': desc.format(q, 'pdns')})
+            if flag == 3: #uri
+                uri = result.get('uri')
+                if uri:
+                    ret.append({'types': ['url'], 'values': [uri], 'comment': desc.format(q, 'uri')})
+            if flag == 4: #samples
+                if type(result) is str:
+                    ret.append({'types': ['sha256'], 'values': [result], 'comment': desc.format(q, 'samples')})
+            if flag == 5: #ssl
+                if type(result) is str:
+                    ret.append({'types': ['x509-fingerprint-sha1'], 'values': [result], 'comment': desc.format(q, 'ssl')})
+            if flag == 6: #reports
+                link = result.get('URL')
+                if link:
+                    ret.append({'types': ['url'], 'values': [link], 'comment': desc.format(q, 'report')})
+
+    return ret
+
+
+def get_hash(q):
+    ret = []
+    for flag in [1, 3, 6, 7]:
+        req = requests.get('https://www.threatminer.org/sample.php', params={'q': q, 'api': 'True', 'rt': flag})
+        if not req.status_code == 200:
+            continue
+        results = req.json().get('results')
+        if not results:
+            continue
+
+        for result in results:
+            if flag == 1: #meta (filename)
+                name = result.get('file_name')
+                if name:
+                    ret.append({'types': ['filename'], 'values': [name], 'comment': desc.format(q, 'file')})
+            if flag == 3: #network
+                domains = result.get('domains')
+                for dom in domains:
+                    if dom.get('domain'):
+                        ret.append({'types': ['domain'], 'values': [dom['domain']], 'comment': desc.format(q, 'network')})
+
+                hosts = result.get('hosts')
+                for h in hosts:
+                    if type(h) is str:
+                        ret.append({'types': ['ip-src', 'ip-dst'], 'values': [h], 'comment': desc.format(q, 'network')})
+            if flag == 6: #detections
+                detections = result.get('av_detections')
+                for d in detections:
+                    if d.get('detection'):
+                        ret.append({'types': ['text'], 'values': [d['detection']], 'comment': desc.format(q, 'detection')})
+            if flag == 7: #report
+                if type(result) is str:
+                    ret.append({'types': ['sha256'], 'values': [result], 'comment': desc.format(q, 'report')})
+
+    return ret
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    return moduleinfo

misp_modules/modules/expansion/virustotal.py

@@ -1,23 +1,24 @@
 import json
 import requests
-import hashlib
+from requests import HTTPError
-import re
 import base64
-import os
 
 misperrors = {'error': 'Error'}
-mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst"],
+mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"],
-                  'output':['domain', "ip-src", "ip-dst", "text"]
+                  'output': ['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep",
+                             "authentihash", "filename"]
                   }
 
 # possible module-types: 'expansion', 'hover' or both
-moduleinfo = {'version': '', 'author': 'Hannah Ward',
+moduleinfo = {'version': '2', 'author': 'Hannah Ward',
               'description': 'Get information from virustotal',
               'module-type': ['expansion']}
 
 # config fields that your code expects from the site admin
 moduleconfig = ["apikey", "event_limit"]
-limit = 5 #Default
+limit = 5  # Default
+comment = '%s: Enriched via VT'
+
 
 def handler(q=False):
     global limit
@@ -30,124 +31,172 @@ def handler(q=False):
     limit = int(q["config"].get("event_limit", 5))
 
     r = {"results": []}
-    
+
     if "ip-src" in q:
-      r["results"] += getIP(q["ip-src"], key)
+        r["results"] += getIP(q["ip-src"], key)
     if "ip-dst" in q:
-      r["results"] += getIP(q["ip-dst"], key)
+        r["results"] += getIP(q["ip-dst"], key)
     if "domain" in q:
-      r["results"] += getDomain(q["domain"], key)
+        r["results"] += getDomain(q["domain"], key)
     if 'hostname' in q:
-      r["results"] += getDomain(q['hostname'], key)
+        r["results"] += getDomain(q['hostname'], key)
+    if 'md5' in q:
+        r["results"] += getHash(q['md5'], key)
+    if 'sha1' in q:
+        r["results"] += getHash(q['sha1'], key)
+    if 'sha256' in q:
+        r["results"] += getHash(q['sha256'], key)
+    if 'sha512' in q:
+        r["results"] += getHash(q['sha512'], key)
 
     uniq = []
     for res in r["results"]:
-      if res not in uniq:
+        if res not in uniq:
-        uniq.append(res)
+            uniq.append(res)
     r["results"] = uniq
     return r
 
-def getIP(ip, key, do_not_recurse = False):
+
+def getHash(hash, key, do_not_recurse=False):
+    req = requests.get("https://www.virustotal.com/vtapi/v2/file/report",
+                       params={"allinfo": 1, "apikey": key, 'resource': hash})
+    try:
+        req.raise_for_status()
+        req = req.json()
+    except HTTPError as e:
+        misperrors['error'] = str(e)
+        return misperrors
+
+    if req["response_code"] == 0:
+        # Nothing found
+        return []
+
+    return getMoreInfo(req, key)
+
+
+def getIP(ip, key, do_not_recurse=False):
     global limit
     toReturn = []
-    req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", 
+    req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report",
-                       params = {"ip":ip, "apikey":key}
+                       params={"ip": ip, "apikey": key})
-                      ).json()
+    try:
+        req.raise_for_status()
+        req = req.json()
+    except HTTPError as e:
+        misperrors['error'] = str(e)
+        return misperrors
+
     if req["response_code"] == 0:
-      #Nothing found
+        # Nothing found
-      return []
+        return []
-    
+
     if "resolutions" in req:
-      for res in req["resolutions"][:limit]:
+        for res in req["resolutions"][:limit]:
-        toReturn.append( {"types":["domain"], "values":[res["hostname"]]})
+            toReturn.append({"types": ["domain"], "values": [res["hostname"]], "comment": comment % ip})
-        #Pivot from here to find all domain info
+            # Pivot from here to find all domain info
-        if not do_not_recurse:
+            if not do_not_recurse:
-          toReturn += getDomain(res["hostname"], key, True)
+                toReturn += getDomain(res["hostname"], key, True)
 
     toReturn += getMoreInfo(req, key)
     return toReturn
-    
+
+
 def getDomain(domain, key, do_not_recurse=False):
     global limit
     toReturn = []
-    req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", 
+    req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report",
-                       params = {"domain":domain, "apikey":key}
+                       params={"domain": domain, "apikey": key})
-                      ).json()
+    try:
+        req.raise_for_status()
+        req = req.json()
+    except HTTPError as e:
+        misperrors['error'] = str(e)
+        return misperrors
+
     if req["response_code"] == 0:
-      #Nothing found
+        # Nothing found
-      return []
+        return []
-    
+
     if "resolutions" in req:
-      for res in req["resolutions"][:limit]:
+        for res in req["resolutions"][:limit]:
-        toReturn.append( {"types":["ip-dst", "ip-src"], "values":[res["ip_address"]]})
+            toReturn.append({"types": ["ip-dst", "ip-src"], "values": [res["ip_address"]], "comment": comment % domain})
-        #Pivot from here to find all info on IPs
+            # Pivot from here to find all info on IPs
-        if not do_not_recurse:
+            if not do_not_recurse:
-          toReturn += getIP(res["ip_address"], key, True)
+                toReturn += getIP(res["ip_address"], key, True)
+    if "subdomains" in req:
+        for subd in req["subdomains"]:
+            toReturn.append({"types": ["domain"], "values": [subd], "comment": comment % domain})
     toReturn += getMoreInfo(req, key)
     return toReturn
 
+
 def findAll(data, keys):
-  a = []
+    a = []
-  if isinstance(data, dict):
+    if isinstance(data, dict):
-    for key in data.keys():
+        for key in data.keys():
-      if key in keys:
+            if key in keys:
-        a.append(data[key])
+                a.append(data[key])
-      else:
+            else:
-        if isinstance(data[key], (dict, list)):
+                if isinstance(data[key], (dict, list)):
-          a += findAll(data[key], keys)
+                    a += findAll(data[key], keys)
-  if isinstance(data, list):
+    if isinstance(data, list):
-    for i in data:
+        for i in data:
-      a += findAll(i, keys)  
+            a += findAll(i, keys)
-    
+
-  return a 
+    return a
+
 
 def getMoreInfo(req, key):
     global limit
     r = []
-    #Get all hashes first
+    # Get all hashes first
     hashes = []
     hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"])
-    r.append({"types":["md5", "sha1", "sha256", "sha512"], "values":hashes})
+    r.append({"types": ["md5", "sha1", "sha256", "sha512"], "values": hashes})
     for hsh in hashes[:limit]:
-      #Search VT for some juicy info
+        # Search VT for some juicy info
-      data = requests.get("http://www.virustotal.com/vtapi/v2/file/report",
+        try:
-                          params={"allinfo":1, "apikey":key, "resource":hsh}
+            data = requests.get("http://www.virustotal.com/vtapi/v2/file/report",
-                         ).json()
+                                params={"allinfo": 1, "apikey": key, "resource": hsh}
+                                ).json()
+        except:
+            continue
 
-      # Go through each key and check if it exists
+        # Go through each key and check if it exists
-      if "submission_names" in data:
+        if "submission_names" in data:
-        r.append({'types':["filename"], "values":data["submission_names"]})
+            r.append({'types': ["filename"], "values": data["submission_names"], "comment": comment % hsh})
 
-      if "ssdeep" in data:
+        if "ssdeep" in data:
-        r.append({'types':["ssdeep"], "values":[data["ssdeep"]]})
+            r.append({'types': ["ssdeep"], "values": [data["ssdeep"]], "comment": comment % hsh})
 
-      if "authentihash" in data:
+        if "authentihash" in data:
-        r.append({"types":["authentihash"], "values":[data["authentihash"]]})
+            r.append({"types": ["authentihash"], "values": [data["authentihash"]], "comment": comment % hsh})
 
-      if "ITW_urls" in data:
+        if "ITW_urls" in data:
-        r.append({"types":["url"], "values":data["ITW_urls"]})
+            r.append({"types": ["url"], "values": data["ITW_urls"], "comment": comment % hsh})
 
-      #Get the malware sample
+        # Get the malware sample
-      sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download",
+        sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download",
-                            params = {"hash":hsh, "apikey":key})
+                              params={"hash": hsh, "apikey": key})
-      
+
-      malsample = sample.content
+        malsample = sample.content
+
+        # It is possible for VT to not give us any submission names
+        if "submission_names" in data:
+            r.append({"types": ["malware-sample"],
+                      "categories": ["Payload delivery"],
+                      "values": data["submission_names"],
+                      "data": str(base64.b64encode(malsample), 'utf-8')
+                      }
+                     )
 
-      # It is possible for VT to not give us any submission names
-      if "submission_names" in data:
-        r.append({"types":["malware-sample"], 
-                  "categories":["Payload delivery"],
-                  "values":data["submission_names"],
-                  "data": str(base64.b64encode(malsample), 'utf-8')
-                  }
-                )
-    
     return r
 
+
 def introspection():
     return mispattributes
 
+
 def version():
     moduleinfo['config'] = moduleconfig
     return moduleinfo
-

misp_modules/modules/expansion/xforceexchange.py

@@ -0,0 +1,101 @@
+import requests
+import json
+import sys
+
+BASEurl = "https://api.xforce.ibmcloud.com/"
+
+extensions = {"ip1": "ipr/%s",
+	      "ip2": "ipr/malware/%s",
+	      "url": "url/%s",
+	      "hash": "malware/%s",
+	      "vuln": "/vulnerabilities/search/%s",
+	      "dns": "resolve/%s"}
+
+sys.path.append('./')
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['ip-src','ip-dst' 'vulnerability', 'md5', 'sha1', 'sha256'], 
+		  'output': ['ip-src', 'ip-dst', 'text', 'domain']}
+
+# possible module-types: 'expansion', 'hover' or both
+moduleinfo = {'version': '1', 'author': 'Joerg Stephan (@johest)',
+              'description': 'IBM X-Force Exchange expansion module',
+              'module-type': ['expansion', 'hover']}
+
+# config fields that your code expects from the site admin
+moduleconfig = ["apikey", "event_limit"]
+limit = 5000 #Default
+
+
+
+def MyHeader(key=False):
+	global limit
+	if key is False:
+		return None
+	
+	return {"Authorization": "Basic %s " % key,
+		   "Accept": "application/json",
+		   'User-Agent': 'Mozilla 5.0'}
+
+def handler(q=False):
+	global limit
+	if q is False:
+		return False
+
+	q = json.loads(q)
+	
+	key = q["config"]["apikey"]
+	limit = int(q["config"].get("event_limit", 5))
+
+	r = {"results": []}
+	
+	if "ip-src" in q:
+		r["results"] += apicall("dns", q["ip-src"], key)
+	if "ip-dst" in q:
+		r["results"] += apicall("dns", q["ip-dst"], key)
+	if "md5" in q:
+		r["results"] += apicall("hash", q["md5"], key)
+	if "sha1" in q:
+		r["results"] += apicall("hash", q["sha1"], key)
+	if "sha256" in q:
+		r["results"] += apicall("hash", q["sha256"], key)  
+	if 'vulnerability' in q:
+		r["results"] += apicall("vuln", q["vulnerability"], key)
+	if "domain" in q:
+                r["results"] += apicall("dns", q["domain"], key)
+
+	uniq = []
+	for res in r["results"]:
+		if res not in uniq:
+			uniq.append(res)
+	r["results"] = uniq
+	return r
+	
+def apicall(indicator_type, indicator, key=False):
+	try:
+		myURL = BASEurl + (extensions[str(indicator_type)])%indicator
+		jsondata = requests.get(myURL, headers=MyHeader(key)).json()
+	except:
+		jsondata = None
+	redata = []
+	#print(jsondata)
+	if not jsondata is None:
+		if indicator_type is "hash":
+			if "malware" in jsondata:
+				lopointer = jsondata["malware"]
+				redata.append({"type": "text", "values": lopointer["risk"]})
+		if indicator_type is "dns":
+			if "records" in str(jsondata):
+				lopointer = jsondata["Passive"]["records"]
+				for dataset in lopointer:
+					redata.append({"type":"domain", "values": dataset["value"]})	
+	
+	return redata
+
+def introspection():
+	return mispattributes
+
+
+def version():
+	moduleinfo['config'] = moduleconfig
+	return moduleinfo

misp_modules/modules/export_mod/__init__.py

@@ -1 +1 @@
-__all__ = ['testexport','cef_export']
+__all__ = ['testexport','cef_export','liteexport']

misp_modules/modules/export_mod/liteexport.py

@@ -0,0 +1,89 @@
+import json
+import base64
+
+misperrors = {'error': 'Error'}
+
+moduleinfo = {'version': '1', 
+			'author': 'TM',
+			'description': 'export lite',
+			'module-type': ['export']}
+
+moduleconfig = ["indent_json_export"]
+
+mispattributes = {}
+outputFileExtension = "json"
+responseType = "application/json"
+
+def handler(q=False):
+	if q is False:
+		return False
+
+	request = json.loads(q)
+
+	config = {}
+	if "config" in request:
+		config = request["config"]
+	else:
+		config = {"indent_json_export" : None}
+
+	if config['indent_json_export'] is not None:
+		try:
+			config['indent_json_export'] = int(config['indent_json_export'])
+		except:
+			config['indent_json_export'] = None
+
+	if 'data' not in request:
+		return False
+
+	#~ Misp json structur
+	liteEvent = {'Event':{}}
+
+	for evt in request['data']:
+		rawEvent = evt['Event']
+		liteEvent['Event']['info'] = rawEvent['info']
+		liteEvent['Event']['Attribute'] = []
+		
+		attrs = evt['Attribute']
+		for attr in attrs:
+			if 'Internal reference' not in attr['category']:
+				liteAttr = {}
+				liteAttr['category'] = attr['category']
+				liteAttr['type'] = attr['type']
+				liteAttr['value'] = attr['value']
+				liteEvent['Event']['Attribute'].append(liteAttr)
+
+	return {'response' : [],
+			'data' : str(base64.b64encode(
+				bytes(
+					json.dumps(liteEvent, indent=config['indent_json_export']),
+					'utf-8')),
+				'utf-8')
+			}
+
+def introspection():
+	modulesetup = {}
+	try:
+		responseType
+		modulesetup['responseType'] = responseType
+	except NameError:
+	  pass
+	try:
+		userConfig
+		modulesetup['userConfig'] = userConfig
+	except NameError:
+		pass
+	try:
+		outputFileExtension
+		modulesetup['outputFileExtension'] = outputFileExtension
+	except NameError:
+		pass
+	try:
+		inputSource
+		modulesetup['inputSource'] = inputSource
+	except NameError:
+		pass
+	return modulesetup
+
+def version():
+	moduleinfo['config'] = moduleconfig
+	return moduleinfo

misp_modules/modules/import_mod/__init__.py

@@ -1,3 +1,4 @@
 from . import _vmray
 
-__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', 'email_import']
+__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport',
+           'email_import', 'mispjson', 'openiocimport']

misp_modules/modules/import_mod/mispjson.py

@@ -0,0 +1,61 @@
+import json
+import base64
+
+misperrors = {'error': 'Error'}
+userConfig = { };
+
+inputSource = ['file']
+
+moduleinfo = {'version': '0.1', 'author': 'Richard van den Berg',
+              'description': 'MISP JSON format import module for merging MISP events',
+              'module-type': ['import']}
+
+moduleconfig = []
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    r = {'results': []}
+    request = json.loads(q)
+    try:
+      mfile = base64.b64decode(request["data"]).decode('utf-8')
+      misp = json.loads(mfile)
+      event = misp['response'][0]['Event']
+      for a in event["Attribute"]:
+        tmp = {}
+        tmp["values"]     = a["value"]
+        tmp["categories"] = a["category"]
+        tmp["types"]      = a["type"]
+        tmp["to_ids"]     = a["to_ids"]
+        tmp["comment"]    = a["comment"]
+        if a.get("data"):
+          tmp["data"]     = a["data"]
+        r['results'].append(tmp)
+    except:
+      pass
+    return r
+
+def introspection():
+    modulesetup = {}
+    try:
+        userConfig
+        modulesetup['userConfig'] = userConfig
+    except NameError:
+        pass
+    try:
+        inputSource
+        modulesetup['inputSource'] = inputSource
+    except NameError:
+        pass
+    return modulesetup
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo
+
+if __name__ == '__main__':
+    x = open('test.json', 'r')
+    r = handler(q=x.read())
+    print(json.dumps(r))

misp_modules/modules/import_mod/openiocimport.py

@@ -0,0 +1,96 @@
+import json
+import base64
+
+from pymisp.tools import openioc
+
+misperrors = {'error': 'Error'}
+userConfig = {
+            'not save ioc': {
+                'type': 'Boolean',
+                'message': 'If you check this box, IOC file will not save as an attachment in MISP'
+            },
+            'default tag': {
+                'type': 'String',
+                'message': 'Add tags spaced by a comma (tlp:white,misp:threat-level="no-risk")',
+                'validation' : '0'
+            }
+        }
+
+inputSource = ['file']
+
+moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot',
+            'description': 'Import OpenIOC package',
+            'module-type': ['import']}
+
+moduleconfig = []
+
+
+def handler(q=False):
+    # Just in case we have no data
+    if q is False:
+        return False
+
+    # The return value
+    r = {'results': []}
+
+    # Load up that JSON
+    q = json.loads(q)
+
+    # It's b64 encoded, so decode that stuff
+    package = base64.b64decode(q.get("data")).decode('utf-8')
+
+    # If something really weird happened
+    if not package:
+        return json.dumps({"success": 0})
+
+    pkg = openioc.load_openioc(package)
+
+    if q.get('config'):
+        if q['config'].get('not save ioc') == "0":
+            addFile = {
+                    "values": [q.get('filename')],
+                    "types": ['attachment'],
+                    "categories": ['Support Tool'],
+                    "data" : q.get('data'),
+                    }
+            # add tag
+            if q['config'].get('default tag') is not None:
+                addFile["tags"] = q['config']['default tag'].split(",")
+            # add file as attachment
+            r["results"].append(addFile)
+
+
+    # return all attributes
+    for attrib in pkg.attributes:
+        toAppend = {
+            "values": [attrib.value],
+            "types": [attrib.type],
+            "categories": [attrib.category],
+            "comment":attrib.comment
+            }
+        # add tag
+        if q.get('config') and q['config'].get('default tag') is not None:
+                toAppend["tags"] = q['config']['default tag'].split(",")
+
+        r["results"].append(toAppend)
+    return r
+
+
+def introspection():
+    modulesetup = {}
+    try:
+        userConfig
+        modulesetup['userConfig'] = userConfig
+    except NameError:
+        pass
+    try:
+        inputSource
+        modulesetup['inputSource'] = inputSource
+    except NameError:
+        pass
+    return modulesetup
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

tests/openioc.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="us-ascii"?>
+<ioc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" id="ea3cab0c-72ad-40cc-abbf-90846fa4afec" last-modified="2011-11-04T19:35:05" xmlns="http://schemas.mandiant.com/2010/ioc">
+  <short_description>STUXNET VIRUS (METHODOLOGY)</short_description>
+  <description>Generic indicator for the stuxnet virus.  When loaded, stuxnet spawns lsass.exe in a suspended state. The malware then maps in its own executable section and fixes up the CONTEXT to point to the newly mapped in section. This is a common task performed by malware and allows the malware to execute under the pretense of a known and trusted process.</description>
+  <keywords>methodology</keywords>
+  <authored_by>Mandiant</authored_by>
+  <authored_date>0001-01-01T00:00:00</authored_date>
+  <links />
+  <definition>
+    <Indicator operator="OR" id="73bc8d65-826b-48d2-b4a8-48918e29e323">
+      <IndicatorItem id="b9ef2559-cc59-4463-81d9-52800545e16e" condition="contains">
+        <Context document="FileItem" search="FileItem/PEInfo/Sections/Section/Name" type="mir" />
+        <Content type="string">.stub</Content>
+      </IndicatorItem>
+      <IndicatorItem id="156bc4b6-a2a1-4735-bfe8-6c8d1f7eae38" condition="contains">
+        <Context document="FileItem" search="FileItem/FileName" type="mir" />
+        <Content type="string">mdmcpq3.PNF</Content>
+      </IndicatorItem>
+      <IndicatorItem id="e57d9a5b-5e6a-41ec-87c8-ee67f3ed2e20" condition="contains">
+        <Context document="FileItem" search="FileItem/FileName" type="mir" />
+        <Content type="string">mdmeric3.PNF</Content>
+      </IndicatorItem>
+      <IndicatorItem id="63d7bee6-b575-4d56-8d43-1c5eac57658f" condition="contains">
+        <Context document="FileItem" search="FileItem/FileName" type="mir" />
+        <Content type="string">oem6C.PNF</Content>
+      </IndicatorItem>
+      <IndicatorItem id="e6bff12a-e23d-45ea-94bd-8289f806bea7" condition="contains">
+        <Context document="FileItem" search="FileItem/FileName" type="mir" />
+        <Content type="string">oem7A.PNF</Content>
+      </IndicatorItem>
+      <Indicator operator="AND" id="422ae9bf-a1ae-41f2-8e54-5b4c6f3e1598">
+        <IndicatorItem id="e93f1610-daaf-4311-bcf3-3aecef8271c0" condition="contains">
+          <Context document="DriverItem" search="DriverItem/DeviceItem/AttachedToDriverName" type="mir" />
+          <Content type="string">fs_rec.sys</Content>
+        </IndicatorItem>
+        <IndicatorItem id="72476f35-8dea-4bae-8239-7c22d05d664f" condition="contains">
+          <Context document="DriverItem" search="DriverItem/DeviceItem/AttachedToDriverName" type="mir" />
+          <Content type="string">mrxsmb.sys</Content>
+        </IndicatorItem>
+        <IndicatorItem id="f98ea5aa-9e23-4f18-b871-b3cf5ba153fe" condition="contains">
+          <Context document="DriverItem" search="DriverItem/DeviceItem/AttachedToDriverName" type="mir" />
+          <Content type="string">sr.sys</Content>
+        </IndicatorItem>
+        <IndicatorItem id="32f61140-0f58-43bc-8cdd-a25db75ca6c4" condition="contains">
+          <Context document="DriverItem" search="DriverItem/DeviceItem/AttachedToDriverName" type="mir" />
+          <Content type="string">fastfat.sys</Content>
+        </IndicatorItem>
+      </Indicator>
+      <Indicator operator="AND" id="eb585bf5-18d8-4837-baf0-80ac74104ca3">
+        <IndicatorItem id="8d85b559-4d18-4e15-b0c9-da5a9b32f53c" condition="contains">
+          <Context document="FileItem" search="FileItem/FileName" type="mir" />
+          <Content type="string">mrxcls.sys</Content>
+        </IndicatorItem>
+        <IndicatorItem id="8a3e425d-fa87-4a31-b20d-8f8630d77933" condition="contains">
+          <Context document="FileItem" search="FileItem/PEInfo/DigitalSignature/CertificateSubject" type="mir" />
+          <Content type="string">Realtek Semiconductor Corp</Content>
+        </IndicatorItem>
+      </Indicator>
+      <Indicator operator="AND" id="bc8d06dd-f879-4609-bb1c-eccded0222ce">
+        <IndicatorItem id="89f194d3-3ee6-4218-93f8-055ea92a9f00" condition="contains">
+          <Context document="FileItem" search="FileItem/FileName" type="mir" />
+          <Content type="string">mrxnet.sys</Content>
+        </IndicatorItem>
+        <IndicatorItem id="c2dae8bf-81b1-49fb-8654-396830d75ade" condition="contains">
+          <Context document="FileItem" search="FileItem/PEInfo/DigitalSignature/CertificateSubject" type="mir" />
+          <Content type="string">Realtek Semiconductor Corp</Content>
+        </IndicatorItem>
+      </Indicator>
+      <Indicator operator="AND" id="00538c36-88fe-42ea-a70f-136a2fb53834">
+        <IndicatorItem id="a779b811-345f-4164-897e-0752837d0c1e" condition="contains">
+          <Context document="RegistryItem" search="RegistryItem/Path" type="mir" />
+          <Content type="string">HKEY_LOCAL_MACHINE\SYSTEM\ControlSet001\Services\MRxCls\ImagePath</Content>
+        </IndicatorItem>
+        <IndicatorItem id="ee981f06-b713-40aa-ac98-c6f4fd82b78d" condition="contains">
+          <Context document="RegistryItem" search="RegistryItem/Text" type="mir" />
+          <Content type="string">mrxcls.sys</Content>
+        </IndicatorItem>
+      </Indicator>
+      <Indicator operator="AND" id="d8d9b32c-f648-4552-9805-93c05ed48219">
+        <IndicatorItem id="c08044e7-e88c-433c-b463-763bdddeff82" condition="contains">
+          <Context document="RegistryItem" search="RegistryItem/Path" type="mir" />
+          <Content type="string">HKEY_LOCAL_MACHINE\SYSTEM\ControlSet001\Services\MRxNet\ImagePath</Content>
+        </IndicatorItem>
+        <IndicatorItem id="38dfb382-ebbe-4685-bbb7-60675b91bd15" condition="contains">
+          <Context document="RegistryItem" search="RegistryItem/Text" type="mir" />
+          <Content type="string">mrxnet.sys</Content>
+        </IndicatorItem>
+      </Indicator>
+    </Indicator>
+  </definition>
+</ioc>

tests/test.py

@@ -42,6 +42,20 @@ class TestModules(unittest.TestCase):
             print(response.json())
             response.connection.close()
 
+    def test_openioc(self):
+        with open("tests/openioc.xml", "rb") as f:
+            content = base64.b64encode(f.read())
+            data = json.dumps({"module": "openiocimport",
+                               "data": content.decode(),
+                               })
+            response = requests.post(self.url + "query", data=data).json()
+            print(response)
+
+            print("OpenIOC :: {}".format(response))
+            values = [x["values"][0] for x in response["results"]]
+            assert("mrxcls.sys" in values)
+            assert("mdmcpq3.PNF" in values)
+
     def test_stix(self):
         with open("tests/stix.xml", "rb") as f:
             content = base64.b64encode(f.read())
@@ -58,7 +72,7 @@ class TestModules(unittest.TestCase):
             assert("eu-society.com" in values)
 
     def test_email_headers(self):
-        query = {"module":"email_import"}
+        query = {"module": "email_import"}
         query["config"] = {"unzip_attachments": None,
                            "guess_zip_attachment_passwords": None,
                            "extract_urls": None}
@@ -106,7 +120,7 @@ class TestModules(unittest.TestCase):
         self.assertIn("<CI7DgL-A6dm92s7gf4-88g@E_0x238G4K2H08H9SDwsw8b6LwuA@mail.example.com>", values)
 
     def test_email_attachment_basic(self):
-        query = {"module":"email_import"}
+        query = {"module": "email_import"}
         query["config"] = {"unzip_attachments": None,
                            "guess_zip_attachment_passwords": None,
                            "extract_urls": None}
@@ -129,9 +143,8 @@ class TestModules(unittest.TestCase):
                 attch_data = base64.b64decode(i["data"])
                 self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-')
 
-
     def test_email_attachment_unpack(self):
-        query = {"module":"email_import"}
+        query = {"module": "email_import"}
         query["config"] = {"unzip_attachments": "true",
                            "guess_zip_attachment_passwords": None,
                            "extract_urls": None}
@@ -163,7 +176,7 @@ class TestModules(unittest.TestCase):
     def test_email_dont_unpack_compressed_doc_attachments(self):
         """Ensures that compressed
         """
-        query = {"module":"email_import"}
+        query = {"module": "email_import"}
         query["config"] = {"unzip_attachments": "true",
                            "guess_zip_attachment_passwords": None,
                            "extract_urls": None}
@@ -193,9 +206,8 @@ class TestModules(unittest.TestCase):
                 self.assertEqual(filesum.hexdigest(),
                                  '098da5381a90d4a51e6b844c18a0fecf2e364813c2f8b317cfdc51c21f2506a5')
 
-
     def test_email_attachment_unpack_with_password(self):
-        query = {"module":"email_import"}
+        query = {"module": "email_import"}
         query["config"] = {"unzip_attachments": "true",
                            "guess_zip_attachment_passwords": 'true',
                            "extract_urls": None}
@@ -222,9 +234,8 @@ class TestModules(unittest.TestCase):
                 self.assertEqual(attch_data,
                                  b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-')
 
-
     def test_email_attachment_password_in_body(self):
-        query = {"module":"email_import"}
+        query = {"module": "email_import"}
         query["config"] = {"unzip_attachments": "true",
                            "guess_zip_attachment_passwords": 'true',
                            "extract_urls": None}
@@ -247,7 +258,7 @@ class TestModules(unittest.TestCase):
                                  'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-')
 
     def test_email_attachment_password_in_body_quotes(self):
-        query = {"module":"email_import"}
+        query = {"module": "email_import"}
         query["config"] = {"unzip_attachments": "true",
                            "guess_zip_attachment_passwords": 'true',
                            "extract_urls": None}
@@ -275,7 +286,7 @@ class TestModules(unittest.TestCase):
                                  'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-')
 
     def test_email_attachment_password_in_html_body(self):
-        query = {"module":"email_import"}
+        query = {"module": "email_import"}
         query["config"] = {"unzip_attachments": "true",
                            "guess_zip_attachment_passwords": 'true',
                            "extract_urls": None}
@@ -305,7 +316,7 @@ class TestModules(unittest.TestCase):
         query['data'] = decode_email(message)
         data = json.dumps(query)
         response = requests.post(self.url + "query", data=data)
-        #print(response.json())
+        # print(response.json())
         values = [x["values"] for x in response.json()["results"]]
         self.assertIn('EICAR.com', values)
         for i in response.json()['results']:
@@ -356,7 +367,7 @@ class TestModules(unittest.TestCase):
                 response = requests.post(self.url + "query", data=data)
 
     def test_email_attachment_password_in_subject(self):
-        query = {"module":"email_import"}
+        query = {"module": "email_import"}
         query["config"] = {"unzip_attachments": "true",
                            "guess_zip_attachment_passwords": 'true',
                            "extract_urls": None}
@@ -385,9 +396,8 @@ class TestModules(unittest.TestCase):
                 self.assertEqual(attch_data,
                                  'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-')
 
-
     def test_email_extract_html_body_urls(self):
-        query = {"module":"email_import"}
+        query = {"module": "email_import"}
         query["config"] = {"unzip_attachments": None,
                            "guess_zip_attachment_passwords": None,
                            "extract_urls": "true"}
@@ -415,12 +425,12 @@ without modifying core components. The API is available via a simple REST API wh
         query['data'] = decode_email(message)
         data = json.dumps(query)
         response = requests.post(self.url + "query", data=data)
-        #print(response.json())
+        # print(response.json())
         values = [x["values"] for x in response.json()["results"]]
         self.assertIn("https://github.com/MISP/MISP", values)
         self.assertIn("https://www.circl.lu/assets/files/misp-training/3.1-MISP-modules.pdf", values)
 
-    #def test_domaintools(self):
+    # def test_domaintools(self):
     #    query = {'config': {'username': 'test_user', 'api_key': 'test_key'}, 'module': 'domaintools', 'domain': 'domaintools.com'}
     #    try:
     #        response = requests.post(self.url + "query", data=json.dumps(query)).json()
@@ -429,33 +439,34 @@ without modifying core components. The API is available via a simple REST API wh
     #    response = requests.post(self.url + "query", data=json.dumps(query)).json()
     #    print(response)
 
+
 def decode_email(message):
     message64 = base64.b64encode(message.as_bytes()).decode()
     return message64
 
 
 def get_base_email():
-    headers = {"Received":"via dmail-2008.19 for +INBOX; Tue, 3 Feb 2009 19:29:12 -0600 (CST)",
+    headers = {"Received": "via dmail-2008.19 for +INBOX; Tue, 3 Feb 2009 19:29:12 -0600 (CST)",
-               "Received":"from abc.luxsci.com ([10.10.10.10]) by xyz.luxsci.com (8.13.7/8.13.7) with ESMTP id n141TCa7022588 for <test@domain.com>; Tue, 3 Feb 2009 19:29:12 -0600",
+               "Received": "from abc.luxsci.com ([10.10.10.10]) by xyz.luxsci.com (8.13.7/8.13.7) with ESMTP id n141TCa7022588 for <test@domain.com>; Tue, 3 Feb 2009 19:29:12 -0600",
-               "Received":"from [192.168.0.3] (verizon.net [44.44.44.44]) (user=test@sender.com mech=PLAIN bits=2) by abc.luxsci.com (8.13.7/8.13.7) with ESMTP id n141SAfo021855 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NOT) for <test@domain.com>; Tue, 3 Feb 2009 19:28:10 -0600",
+               "Received": "from [192.168.0.3] (verizon.net [44.44.44.44]) (user=test@sender.com mech=PLAIN bits=2) by abc.luxsci.com (8.13.7/8.13.7) with ESMTP id n141SAfo021855 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NOT) for <test@domain.com>; Tue, 3 Feb 2009 19:28:10 -0600",
-               "X-Received":"by 192.168.0.45 with SMTP id q4mr156123401yw1g.911.1912342394963; Tue, 3 Feb 2009 19:32:15 -0600 (PST)",
+               "X-Received": "by 192.168.0.45 with SMTP id q4mr156123401yw1g.911.1912342394963; Tue, 3 Feb 2009 19:32:15 -0600 (PST)",
-               "Message-ID":"<4988EF2D.40804@example.com>",
+               "Message-ID": "<4988EF2D.40804@example.com>",
-               "Date":"Tue, 03 Feb 2009 20:28:13 -0500",
+               "Date": "Tue, 03 Feb 2009 20:28:13 -0500",
-               "From":'"Innocent Person" <IgnoreMeImInnocent@sender.com>',
+               "From": '"Innocent Person" <IgnoreMeImInnocent@sender.com>',
-               "User-Agent":'Thunderbird 2.0.0.19 (Windows/20081209)',
+               "User-Agent": 'Thunderbird 2.0.0.19 (Windows/20081209)',
-               "Sender":'"Malicious MailAgent" <mailagent@example.com>',
+               "Sender": '"Malicious MailAgent" <mailagent@example.com>',
-               "References":"<CI7DgL-A6dm92s7gf4-88g@E_0x238G4K2H08H9SDwsw8b6LwuA@mail.example.com>",
+               "References": "<CI7DgL-A6dm92s7gf4-88g@E_0x238G4K2H08H9SDwsw8b6LwuA@mail.example.com>",
-               "In-Reply-To":"<CI7DgL-A6dm92s7gf4-88g@E_0x238G4K2H08H9SDwsw8b6LwuA@mail.example.com>",
+               "In-Reply-To": "<CI7DgL-A6dm92s7gf4-88g@E_0x238G4K2H08H9SDwsw8b6LwuA@mail.example.com>",
-               "Accept-Language":'en-US',
+               "Accept-Language": 'en-US',
-               "X-Mailer":'mlx 5.1.7',
+               "X-Mailer": 'mlx 5.1.7',
                "Return-Path": "evil_spoofer@example.com",
-               "Thread-Topic":'This is a thread.',
+               "Thread-Topic": 'This is a thread.',
-               "Thread-Index":'AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==',
+               "Thread-Index": 'AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==',
-               "Content-Language":'en-US',
+               "Content-Language": 'en-US',
-               "To":'"Testy Testerson" <test@domain.com>',
+               "To": '"Testy Testerson" <test@domain.com>',
-               "Cc":'"Second Person" <second@domain.com>, "Other Friend" <other@friend.net>, "Last One" <last_one@finally.com>',
+               "Cc": '"Second Person" <second@domain.com>, "Other Friend" <other@friend.net>, "Last One" <last_one@finally.com>',
-               "Subject":'Example Message',
+               "Subject": 'Example Message',
-               "MIME-Version":'1.0'}
+               "MIME-Version": '1.0'}
     msg = MIMEMultipart()
     for key, val in headers.items():
         msg.add_header(key, val)