From 4bc76acd37b6403bd45c4039ef28847c5bbbdb5c Mon Sep 17 00:00:00 2001
From: Andreas Muehlemann <andreas.muehlemann@switch.ch>
Date: Wed, 28 Sep 2016 14:05:43 +0200
Subject: [PATCH 01/54] added geoip_country.py

---
 .../modules/expansion/geoip_country.py        | 66 +++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 misp_modules/modules/expansion/geoip_country.py

diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py
new file mode 100644
index 0000000..01e95de
--- /dev/null
+++ b/misp_modules/modules/expansion/geoip_country.py
@@ -0,0 +1,66 @@
+import json, pygeoip
+import sys, logging
+
+log = logging.getLogger('geoip_country')
+log.setLevel(logging.DEBUG)
+ch = logging.StreamHandler(sys.stdout)
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ch.setFormatter(formatter)
+log.addHandler(ch)
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['freetext']}
+
+# possible module-types: 'expansion', 'hover' or both
+moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
+              'description': 'Query a local copy of Maxminds Geolite database',
+              'module-type': ['expansion', 'hover']}
+
+# config fields that your code expects from the site admin
+moduleconfig = ['database']
+
+# get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
+gi = pygeoip.GeoIP('/opt/misp-modules/var/GeoIP.dat')
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+
+    log.debug("BLA")
+
+    if request.get('ip-dst'):
+        toquery = request['ip-dst']
+    elif request.get('ip-src'):
+        toquery = request['ip-src']
+    elif request.get('domain|ip'):
+        toquery = request['domain|ip'].split('|')[1]
+    else:
+        return false
+
+    log.debug(toquery)
+
+    #if request.get('config'):
+    #    if request['config'].get('database'):
+    #        gi = pygeoip.GeoIP(request['config'].get('database'))
+    #else:
+    #    gi = pygeoip.GeoIP('/opt/misp-modules/var/GeoIP.dat')
+
+    try:
+        answer = gi.country_code_by_addr(toquery)
+    except:
+        misperrors['error'] = "GeoIP resolving error"
+        return misperrors
+
+    r = {'results': [{'types': mispattributes['output'],
+                      'values': [str(answer)]}]}
+                     
+    return r
+
+def introspection():
+    return mispattributes
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

From a568d1a1b3aa992256b08b5e68c092c2e49080f3 Mon Sep 17 00:00:00 2001
From: Andreas Muehlemann <andreas.muehlemann@switch.ch>
Date: Wed, 28 Sep 2016 14:06:18 +0200
Subject: [PATCH 02/54] updated geoip_country to __init__.py

---
 misp_modules/modules/expansion/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py
index e3b8d67..8ed3f35 100644
--- a/misp_modules/modules/expansion/__init__.py
+++ b/misp_modules/modules/expansion/__init__.py
@@ -1,2 +1,3 @@
 __all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
-           'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns']
+           'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns'
+           'geoip_country']

From 42fc574f85217dcfe724f28631a70e2eb46915a1 Mon Sep 17 00:00:00 2001
From: Andreas Muehlemann <andreas.muehlemann@switch.ch>
Date: Wed, 28 Sep 2016 17:00:11 +0200
Subject: [PATCH 03/54] added pygeoip to the REQUIREMENTS list

---
 REQUIREMENTS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/REQUIREMENTS b/REQUIREMENTS
index f138191..fd7f901 100644
--- a/REQUIREMENTS
+++ b/REQUIREMENTS
@@ -14,3 +14,4 @@ asnhistory
 git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client
 pillow
 pytesseract
+pygeoip

From 03b6fd7b74d35e07d2066eb3bb155d2127a401f1 Mon Sep 17 00:00:00 2001
From: Roman Graf <roman.graf@ait.ac.at>
Date: Tue, 11 Oct 2016 14:48:59 +0200
Subject: [PATCH 04/54] label replaced by text, which is existing attribute

---
 misp_modules/modules/expansion/wiki.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/misp_modules/modules/expansion/wiki.py b/misp_modules/modules/expansion/wiki.py
index 579392d..85279b1 100755
--- a/misp_modules/modules/expansion/wiki.py
+++ b/misp_modules/modules/expansion/wiki.py
@@ -3,10 +3,10 @@ import requests
 from SPARQLWrapper import SPARQLWrapper, JSON
 
 misperrors = {'error': 'Error'}
-mispattributes = {'input': ['label'], 'output': ['text']}
-moduleinfo = {'version': '0.1', 'author': 'Roman Graf', 'description': 'An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.', 'module-type': ['hover']}
+mispattributes = {'input': ['text'], 'output': ['text']}
+moduleinfo = {'version': '0.2', 'author': 'Roman Graf', 'description': 'An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.', 'module-type': ['hover']}
 moduleconfig = []
-# sample query label 'Microsoft' should provide Wikidata link https://www.wikidata.org/wiki/Q2283 in response
+# sample query text 'Microsoft' should provide Wikidata link https://www.wikidata.org/wiki/Q2283 in response
 wiki_api_url = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql'
 
 
@@ -14,15 +14,15 @@ def handler(q=False):
     if q is False:
         return False
     request = json.loads(q)
-    if not request.get('label'):
-        misperrors['error'] = 'Query label missing'
+    if not request.get('text'):
+        misperrors['error'] = 'Query text missing'
         return misperrors
     
     sparql = SPARQLWrapper(wiki_api_url)
     query_string = \
 	"SELECT ?item \n" \
 	"WHERE { \n" \
-	"?item rdfs:label\"" + request.get('label') + "\" @en \n" \
+	"?item rdfs:label\"" + request.get('text') + "\" @en \n" \
 	"}\n";
     sparql.setQuery(query_string)
     sparql.setReturnFormat(JSON)

From adda9562c080dc4d493f07a6a9ee51288eee8a13 Mon Sep 17 00:00:00 2001
From: Koen Van Impe <koen.vanimpe@cudeso.be>
Date: Sun, 13 Nov 2016 21:43:59 +0100
Subject: [PATCH 05/54] VMRay Import & Submit module

* First commit
* No support for archives (yet) submit
---
 misp_modules/modules/expansion/__init__.py    |   2 +-
 .../modules/expansion/vmray_rest_api.py       | 152 +++++++++
 .../modules/expansion/vmray_submit.py         | 152 +++++++++
 misp_modules/modules/import_mod/__init__.py   |   2 +-
 .../modules/import_mod/vmray_import.py        | 300 ++++++++++++++++++
 .../modules/import_mod/vmray_rest_api.py      | 152 +++++++++
 6 files changed, 758 insertions(+), 2 deletions(-)
 create mode 100644 misp_modules/modules/expansion/vmray_rest_api.py
 create mode 100644 misp_modules/modules/expansion/vmray_submit.py
 create mode 100644 misp_modules/modules/import_mod/vmray_import.py
 create mode 100644 misp_modules/modules/import_mod/vmray_rest_api.py

diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py
index 587b5d5..fdbdfb8 100644
--- a/misp_modules/modules/expansion/__init__.py
+++ b/misp_modules/modules/expansion/__init__.py
@@ -1,2 +1,2 @@
-__all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
+__all__ = ['vmray_submit','asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
            'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'wiki']
diff --git a/misp_modules/modules/expansion/vmray_rest_api.py b/misp_modules/modules/expansion/vmray_rest_api.py
new file mode 100644
index 0000000..64b8024
--- /dev/null
+++ b/misp_modules/modules/expansion/vmray_rest_api.py
@@ -0,0 +1,152 @@
+#!/usr/bin/python3
+"""Python client library for VMRay REST API"""
+
+import base64
+import datetime
+import os.path
+import requests
+#import urlparse
+import urllib.parse
+
+from io import IOBase
+from io import BytesIO
+
+# disable nasty certification warning
+# pylint: disable=no-member
+try:
+    requests.packages.urllib3.disable_warnings()
+except AttributeError:
+    try:
+        import urllib3
+        try:
+            urllib3.disable_warnings()
+        except AttributeError:
+            pass
+    except ImportError:
+        pass
+
+# pylint: disable=
+
+
+class VMRayRESTAPIError(Exception):
+    """Exception class that is used when API returns an error"""
+
+    def __init__(self, *args, **kwargs):
+        self.status_code = kwargs.pop("status_code", None)
+        Exception.__init__(self, *args, **kwargs)
+
+
+def handle_rest_api_result(result):
+    """Handle result of API request (check for errors)"""
+
+    if (result.status_code < 200) or (result.status_code > 299):
+        try:
+            json_result = result.json()
+        except ValueError:
+            raise VMRayRESTAPIError("API returned error %u: %s" % (result.status_code, result.text), status_code=result.status_code)
+
+        raise VMRayRESTAPIError(json_result.get("error_msg", "Unknown error"), status_code=result.status_code)
+
+
+class VMRayRESTAPI(object):
+    """VMRay REST API class"""
+
+    def __init__(self, server, api_key, verify_cert=True):
+        # split server URL into components
+        url_desc = urllib.parse.urlsplit(server)
+
+        # assume HTTPS if no scheme is specified
+        if url_desc.scheme == "":
+            server = "https://" + server
+
+        # save variables
+        self.server = server
+        self.api_key = api_key
+        self.verify_cert = verify_cert
+
+    def call(self, http_method, api_path, params=None, raw_data=False):
+        """Call VMRay REST API"""
+
+        # get function of requests package
+        requests_func = getattr(requests, http_method.lower())
+
+        # parse parameters
+        req_params = {}
+        file_params = {}
+
+        if params is not None:
+            for key, value in params.items():
+                if isinstance(value, (datetime.date,
+                                      datetime.datetime,
+                                      float,
+                                      int)):
+                    req_params[key] = str(value)
+                elif isinstance(value, str):
+                    req_params[key] = str(value)
+                elif isinstance(value, dict):
+                    filename = value["filename"]
+                    sample = value["data"]
+                    file_params[key] = (filename, sample, "application/octet-stream")
+                elif isinstance(value, file) or hasattr(value, "read"):
+                    filename = os.path.split(value.name)[1]
+                    # For the following block refer to DEV-1820
+                    try:
+                        filename.decode("ASCII")
+                    except (UnicodeDecodeError, UnicodeEncodeError):
+                        b64_key = key + "name_b64enc"
+                        byte_value = filename.encode("utf-8")
+                        b64_value = base64.b64encode(byte_value)
+
+                        filename = "@param=%s" % b64_key
+                        req_params[b64_key] = b64_value
+                    file_params[key] = (filename, value, "application/octet-stream")
+                else:
+                    raise VMRayRESTAPIError("Parameter \"%s\" has unknown type \"%s\"" % (key, type(value)))
+
+        # construct request
+        if file_params:
+            files = file_params
+        else:
+            files = None
+
+        # we need to adjust some stuff for POST requests
+        if http_method.lower() == "post":
+            req_data = req_params
+            req_params = None
+        else:
+            req_data = None
+
+        # do request
+        result = requests_func(self.server + api_path, data=req_data, params=req_params, headers={"Authorization": "api_key " + self.api_key}, files=files, verify=self.verify_cert, stream=raw_data)
+        handle_rest_api_result(result)
+
+        if raw_data:
+            return result.raw
+
+        # parse result
+        try:
+            json_result = result.json()
+        except ValueError:
+            raise ValueError("API returned invalid JSON: %s" % (result.text))
+
+        # if there are no cached elements then return the data
+        if "continuation_id" not in json_result:
+            return json_result.get("data", None)
+
+        data = json_result["data"]
+
+        # get cached results
+        while "continuation_id" in json_result:
+            # send request to server
+            result = requests.get("%s/rest/continuation/%u" % (self.server, json_result["continuation_id"]), headers={"Authorization": "api_key " + self.api_key}, verify=self.verify_cert)
+            handle_rest_api_result(result)
+
+            # parse result
+            try:
+                json_result = result.json()
+            except ValueError:
+                raise ValueError("API returned invalid JSON: %s" % (result.text))
+
+            data.extend(json_result["data"])
+
+        return data
diff --git a/misp_modules/modules/expansion/vmray_submit.py b/misp_modules/modules/expansion/vmray_submit.py
new file mode 100644
index 0000000..9ab4e05
--- /dev/null
+++ b/misp_modules/modules/expansion/vmray_submit.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+
+'''
+Submit sample to  VMRay.
+
+Submit a sample to VMRay
+
+TODO:
+ # Deal with malicious samples (ZIP file, 'infected')
+ # Deal with archive submissions
+
+'''
+
+import json
+import re
+import base64
+
+import sys
+import os
+base_dir = os.path.dirname(__file__) or '.'
+sys.path.append(base_dir)
+from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError
+import io
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['attachment'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']}
+moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe',
+              'description': 'Submit a sample to VMRay',
+              'module-type': ['expansion']}
+moduleconfig = ['apikey', 'url', 'shareable', 'do_not_reanalyze', 'do_not_include_vmrayjobids']
+
+
+include_vmrayjobids = False
+
+
+def handler(q=False):
+    global include_vmrayjobids
+
+    if q is False:
+        return False
+    request = json.loads(q)
+
+    try:
+        data = request.get("data")
+        attachment = request.get("attachment")
+        data = base64.b64decode(data)
+    except:
+        misperrors['error'] = "Unable to process submited sample data"
+        return misperrors
+
+    if (request["config"].get("apikey") is None) or (request["config"].get("url") is None):
+        misperrors["error"] = "Missing API key or server URL (hint: try cloud.vmray.com)"
+        return misperrors
+
+    api = VMRayRESTAPI(request["config"].get("url"), request["config"].get("apikey"), False)
+
+    shareable = request["config"].get("shareable")
+    do_not_reanalyze = request["config"].get("do_not_reanalyze")
+    do_not_include_vmrayjobids = request["config"].get("do_not_include_vmrayjobids")
+
+    # Do we want the sample to be shared?
+    if shareable == "True":
+        shareable = True
+    else:
+        shareable = False
+
+    # Always reanalyze the sample?
+    if do_not_reanalyze == "True":
+        do_not_reanalyze = True
+    else:
+        do_not_reanalyze = False
+    reanalyze = not do_not_reanalyze
+
+    # Include the references to VMRay job IDs
+    if do_not_include_vmrayjobids == "True":
+        do_not_include_vmrayjobids = True
+    else:
+        do_not_include_vmrayjobids = False
+    include_vmrayjobids = not do_not_include_vmrayjobids
+
+    if data and attachment:
+        args = {}
+        args["shareable"] = shareable
+        args["sample_file"] = {'data': io.BytesIO( data ) , 'filename': attachment }
+        args["reanalyze"] = reanalyze
+
+        try:
+            vmraydata = vmraySubmit(api, args)
+            if vmraydata["errors"]:
+                misperrors['error'] = "VMRay: %s" % vmraydata["errors"][0]["error_msg"]
+                return misperrors
+            else:
+                return vmrayProcess(vmraydata)
+        except:
+            misperrors['error'] = "Problem when calling API."
+            return misperrors
+    else:
+        misperrors['error'] = "No sample data or filename."
+        return misperrors
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo
+
+
+def vmrayProcess(vmraydata):
+    ''' Process the JSON file returned by vmray'''
+    if vmraydata:
+        try:
+            submissions = vmraydata["submissions"][0]
+            jobs = vmraydata["jobs"]
+
+            # Result received?
+            if submissions and jobs:
+                r = {'results': []}
+                r["results"].append( {"types": "md5", "values": submissions["submission_sample_md5"]} )
+                r["results"].append( {"types": "sha1", "values": submissions["submission_sample_sha1"]} )
+                r["results"].append( {"types": "sha256", "values": submissions["submission_sample_sha256"]} )
+                r["results"].append( {"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]} )
+                r["results"].append( {"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]} )
+                r["results"].append( {"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]} )
+                r["results"].append( {"types": "link", "values": submissions["submission_webif_url"]} )
+
+                # Include data from different jobs
+                if include_vmrayjobids:
+                    for job in jobs:
+                        job_id = job["job_id"]
+                        job_vm_name = job["job_vm_name"]
+                        job_configuration_name = job["job_configuration_name"]
+                        r["results"].append( {"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name) })
+                return r
+            else:
+                misperrors['error'] = "No valid results returned."
+                return misperrors
+        except:
+            misperrors['error'] = "No valid submission data returned."
+            return misperrors
+    else:
+        misperrors['error'] = "Unable to parse results."
+        return misperrors
+
+
+def vmraySubmit(api, args):
+    ''' Submit the sample to VMRay'''
+    vmraydata = api.call("POST", "/rest/sample/submit", args)
+    return vmraydata
+
diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py
index b53259e..4b30a1a 100644
--- a/misp_modules/modules/import_mod/__init__.py
+++ b/misp_modules/modules/import_mod/__init__.py
@@ -1 +1 @@
-__all__ = ['testimport', 'ocr', 'stiximport']
+__all__ = ['vmray_import','testimport', 'ocr', 'stiximport']
diff --git a/misp_modules/modules/import_mod/vmray_import.py b/misp_modules/modules/import_mod/vmray_import.py
new file mode 100644
index 0000000..397bd39
--- /dev/null
+++ b/misp_modules/modules/import_mod/vmray_import.py
@@ -0,0 +1,300 @@
+#!/usr/bin/env python3
+
+'''
+Import VMRay results.
+
+This version supports import from different analyze jobs, starting from one sample
+(the supplied sample_id).
+
+Requires "vmray_rest_api"
+
+TODO:
+ # Import one job (analyze_id)
+ # Import STIX package (XML version)
+
+'''
+
+import json
+import re
+import sys
+import os
+base_dir = os.path.dirname(__file__) or '.'
+sys.path.append(base_dir)
+from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError
+
+misperrors = {'error': 'Error'}
+inputSource = []
+moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe',
+              'description': 'Import VMRay (VTI) results',
+              'module-type': ['import']}
+userConfig = {
+               'include_textdescr': {
+                 'type': 'Boolean',
+                 'message': 'Include textual description'
+               },
+               'include_analysisid': {
+                 'type': 'Boolean',
+                 'message': 'Include VMRay analysis_id text'
+               },
+               'only_network_info': {
+                 'type': 'Boolean',
+                 'message': 'Only include network (src-ip, hostname, domain, ...) information'
+               },
+               'sample_id': {
+                 'type': 'Integer',
+                 'errorMessage': 'Expected a sample ID',
+                 'message': 'The VMRay sample_id'
+               }
+             };
+moduleconfig = ['apikey', 'url']
+
+include_textdescr = False
+include_analysisid = False
+only_network_info = False
+
+def handler(q=False):
+    global include_textdescr
+    global include_analysisid
+    global only_network_info
+
+    if q is False:
+        return False
+    request = json.loads(q)
+
+    include_textdescr = request["config"].get("include_textdescr")
+    include_analysisid = request["config"].get("include_analysisid")
+    only_network_info = request["config"].get("only_network_info")
+    if include_textdescr == "1":
+        include_textdescr = True
+    else:
+        include_textdescr = False
+    if include_analysisid == "1":
+        include_analysisid = True
+    else:
+        include_analysisid = False
+    if only_network_info == "1":
+        only_network_info = True
+    else:
+        only_network_info = False
+
+    sample_id = int(request["config"].get("sample_id"))
+
+    if (request["config"].get("apikey") is None) or (request["config"].get("url") is None):
+        misperrors["error"] = "Missing API key or server URL (hint: try cloud.vmray.com)"
+        return misperrors
+
+    if sample_id > 0:
+        try:
+            api = VMRayRESTAPI(request["config"].get("url"), request["config"].get("apikey"), False)
+            vmray_results = {'results': []}
+            # Get all information on the sample, returns a set of finished analyze jobs
+            data = vmrayGetInfoAnalysis(api, sample_id)
+            if data["data"]:
+                for analysis in data["data"]:
+                    analysis_id = analysis["analysis_id"]
+
+                    if analysis_id > 0:
+                        # Get the details for an analyze job
+                        analysis_data = vmrayDownloadAnalysis(api, analysis_id)
+
+                        if analysis_data:
+                            p = vmrayVtiPatterns(analysis_data["vti_patterns"])
+                            if p:
+                                if include_analysisid:
+                                    url1 = "https://cloud.vmray.com/user/analysis/view?from_sample_id=%u" % sample_id
+                                    url2 = "&id=%u" % analysis_id
+                                    url3 = "&sub=%2Freport%2Foverview.html"
+                                    p["results"].append({ "values": url1 + url2 + url3, "types": "link" })
+                                vmray_results = {'results': vmray_results["results"] + p["results"] }
+
+                # Clean up (remove doubles)
+                vmray_results = vmrayCleanup(vmray_results)
+                return vmray_results
+            else:
+                misperrors['error'] = "Unable to fetch sample id %u" % (sample_id)
+                return misperrors
+        except:
+            misperrors['error'] = "Unable to access VMRay API"
+            return misperrors
+    else:
+        misperrors['error'] = "Not a valid sample id"
+        return misperrors
+
+
+
+def introspection():
+    modulesetup = {}
+    try:
+        userConfig
+        modulesetup['userConfig'] = userConfig
+    except NameError:
+        pass
+    try:
+        inputSource
+        modulesetup['inputSource'] = inputSource
+    except NameError:
+        pass
+    return modulesetup
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo
+
+
+def vmrayGetInfoAnalysis(api, sample_id):
+    ''' Get information from a sample, returns a set of analyzed reports'''
+
+    if sample_id:
+        data = api.call("GET", "/rest/analysis/sample/%u" % (sample_id), raw_data=True)
+        return json.loads(data.read().decode())
+    else:
+        return False
+
+
+def vmrayDownloadAnalysis(api, analysis_id):
+    ''' Get the details from an analysis'''
+    if analysis_id:
+        data = api.call("GET", "/rest/analysis/%u/archive/additional/vti_result.json" % (analysis_id), raw_data=True)
+        return json.loads(data.read().decode())
+    else:
+        return False
+
+
+def vmrayVtiPatterns(vti_patterns):
+    ''' Match the VTI patterns to MISP data'''
+
+    if vti_patterns:
+        r = {'results': []}
+        y = {'results': []}
+
+        for pattern in vti_patterns:
+            content = False
+            if pattern["category"] == "_network" and pattern["operation"] == "_download_data":
+                content = vmrayGeneric(pattern, "url", 1)
+            elif pattern["category"] == "_network" and pattern["operation"] == "_connect":
+                content = vmrayConnect(pattern)
+
+            elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page":
+                content = vmrayGeneric(pattern)
+            elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint":
+                content = vmrayGeneric(pattern, "mutex", 1)
+
+            elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
+                content = vmrayGeneric(pattern)
+            elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage":
+                content = vmrayGeneric(pattern)
+
+            elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file":
+                content = vmrayGeneric(pattern, "filename", 1)
+            elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file":
+                content = vmrayGeneric(pattern, "filename", 1)
+
+            elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory":
+                content = vmrayGeneric(pattern)
+            elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow":
+                content = vmrayGeneric(pattern)
+            elif only_network_info == False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files":
+                content = vmrayGeneric(pattern)
+
+            elif only_network_info == False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script":
+                content = vmrayGeneric(pattern, "regkey", 1)
+            elif only_network_info == False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges":
+                content = vmrayGeneric(pattern)
+
+            if content:
+                r["results"].append( content["attributes"] )
+                r["results"].append( content["text"] )
+
+        # Remove empty results
+        r["results"] = [x for x in r["results"] if isinstance(x, dict) and  len(x["values"]) != 0]
+        for el in r["results"]:
+            if not el in y["results"]:
+                y["results"].append( el )
+        return y
+    else:
+        return False
+
+
+def vmrayCleanup(x):
+    ''' Remove doubles'''
+    y = {'results': []}
+
+    for el in x["results"]:
+        if not el in y["results"]:
+            y["results"].append( el )
+    return y
+
+
+def vmraySanitizeInput(s):
+    ''' Sanitize some input so it gets properly imported in MISP'''
+    if s:
+        s = s.replace('"','')
+        s =  re.sub('\\\\', r'\\', s)
+        return s
+    else:
+        return False
+
+
+def vmrayGeneric(el, attr = "", attrpos = 1):
+    ''' Convert a 'generic' VTI pattern to MISP data'''
+
+    r = {"values": []}
+    f = {"values": []}
+
+    if el:
+        content = el["technique_desc"]
+        if content:
+            if attr:
+                content_split = content.split("\"")
+                content_split[attrpos] = vmraySanitizeInput(content_split[attrpos])
+                r["values"].append(content_split[attrpos])
+                r["types"] = [attr]
+
+            # Adding the value also as text to get the extra description,
+            # but this is pretty useless for "url"
+            if include_textdescr and attr != "url":
+                f["values"].append(vmraySanitizeInput(content))
+                f["types"] = ["text"]
+
+            return {    "text": f,
+                        "attributes": r}
+        else:
+            return False
+    else:
+        return False
+
+
+def vmrayConnect(el):
+    ''' Extension of vmrayGeneric , parse network connect data'''
+    ipre = re.compile("([0-9]{1,3}.){3}[0-9]{1,3}")
+
+    r = {"values": []}
+    f = {"values": []}
+
+    if el:
+        content = el["technique_desc"]
+        if content:
+            target = content.split("\"")
+            port = (target[1].split(":"))[1]
+            host = (target[1].split(":"))[0]
+            if ipre.match(str(host)):
+                r["values"].append(host)
+                r["types"] = ["ip-dst"]
+            else:
+                r["values"].append(host)
+                r["types"] = ["domain", "hostname"]
+
+            f["values"].append(vmraySanitizeInput(target[1]))
+            f["types"] = ["text"]
+
+            if include_textdescr:
+                f["values"].append(vmraySanitizeInput(content))
+                f["types"] = ["text"]
+
+            return {    "text": f,
+                        "attributes": r}
+        else:
+            return False
+    else:
+        return False
diff --git a/misp_modules/modules/import_mod/vmray_rest_api.py b/misp_modules/modules/import_mod/vmray_rest_api.py
new file mode 100644
index 0000000..64b8024
--- /dev/null
+++ b/misp_modules/modules/import_mod/vmray_rest_api.py
@@ -0,0 +1,152 @@
+#!/usr/bin/python3
+"""Python client library for VMRay REST API"""
+
+import base64
+import datetime
+import os.path
+import requests
+#import urlparse
+import urllib.parse
+
+from io import IOBase
+from io import BytesIO
+
+# disable nasty certification warning
+# pylint: disable=no-member
+try:
+    requests.packages.urllib3.disable_warnings()
+except AttributeError:
+    try:
+        import urllib3
+        try:
+            urllib3.disable_warnings()
+        except AttributeError:
+            pass
+    except ImportError:
+        pass
+
+# pylint: disable=
+
+
+class VMRayRESTAPIError(Exception):
+    """Exception class that is used when API returns an error"""
+
+    def __init__(self, *args, **kwargs):
+        self.status_code = kwargs.pop("status_code", None)
+        Exception.__init__(self, *args, **kwargs)
+
+
+def handle_rest_api_result(result):
+    """Handle result of API request (check for errors)"""
+
+    if (result.status_code < 200) or (result.status_code > 299):
+        try:
+            json_result = result.json()
+        except ValueError:
+            raise VMRayRESTAPIError("API returned error %u: %s" % (result.status_code, result.text), status_code=result.status_code)
+
+        raise VMRayRESTAPIError(json_result.get("error_msg", "Unknown error"), status_code=result.status_code)
+
+
+class VMRayRESTAPI(object):
+    """VMRay REST API class"""
+
+    def __init__(self, server, api_key, verify_cert=True):
+        # split server URL into components
+        url_desc = urllib.parse.urlsplit(server)
+
+        # assume HTTPS if no scheme is specified
+        if url_desc.scheme == "":
+            server = "https://" + server
+
+        # save variables
+        self.server = server
+        self.api_key = api_key
+        self.verify_cert = verify_cert
+
+    def call(self, http_method, api_path, params=None, raw_data=False):
+        """Call VMRay REST API"""
+
+        # get function of requests package
+        requests_func = getattr(requests, http_method.lower())
+
+        # parse parameters
+        req_params = {}
+        file_params = {}
+
+        if params is not None:
+            for key, value in params.items():
+                if isinstance(value, (datetime.date,
+                                      datetime.datetime,
+                                      float,
+                                      int)):
+                    req_params[key] = str(value)
+                elif isinstance(value, str):
+                    req_params[key] = str(value)
+                elif isinstance(value, dict):
+                    filename = value["filename"]
+                    sample = value["data"]
+                    file_params[key] = (filename, sample, "application/octet-stream")
+                elif isinstance(value, file) or hasattr(value, "read"):
+                    filename = os.path.split(value.name)[1]
+                    # For the following block refer to DEV-1820
+                    try:
+                        filename.decode("ASCII")
+                    except (UnicodeDecodeError, UnicodeEncodeError):
+                        b64_key = key + "name_b64enc"
+                        byte_value = filename.encode("utf-8")
+                        b64_value = base64.b64encode(byte_value)
+
+                        filename = "@param=%s" % b64_key
+                        req_params[b64_key] = b64_value
+                    file_params[key] = (filename, value, "application/octet-stream")
+                else:
+                    raise VMRayRESTAPIError("Parameter \"%s\" has unknown type \"%s\"" % (key, type(value)))
+
+        # construct request
+        if file_params:
+            files = file_params
+        else:
+            files = None
+
+        # we need to adjust some stuff for POST requests
+        if http_method.lower() == "post":
+            req_data = req_params
+            req_params = None
+        else:
+            req_data = None
+
+        # do request
+        result = requests_func(self.server + api_path, data=req_data, params=req_params, headers={"Authorization": "api_key " + self.api_key}, files=files, verify=self.verify_cert, stream=raw_data)
+        handle_rest_api_result(result)
+
+        if raw_data:
+            return result.raw
+
+        # parse result
+        try:
+            json_result = result.json()
+        except ValueError:
+            raise ValueError("API returned invalid JSON: %s" % (result.text))
+
+        # if there are no cached elements then return the data
+        if "continuation_id" not in json_result:
+            return json_result.get("data", None)
+
+        data = json_result["data"]
+
+        # get cached results
+        while "continuation_id" in json_result:
+            # send request to server
+            result = requests.get("%s/rest/continuation/%u" % (self.server, json_result["continuation_id"]), headers={"Authorization": "api_key " + self.api_key}, verify=self.verify_cert)
+            handle_rest_api_result(result)
+
+            # parse result
+            try:
+                json_result = result.json()
+            except ValueError:
+                raise ValueError("API returned invalid JSON: %s" % (result.text))
+
+            data.extend(json_result["data"])
+
+        return data

From c676587461b0ca102449ee474f18c28fd935b79d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Tue, 15 Nov 2016 16:43:11 +0100
Subject: [PATCH 06/54] Multiple clanges in the vmray modules.

* Generic fix to load modules requiring a local library
* Fix python3 support
* PEP8 related cleanups
---
 misp_modules/__init__.py                      |  4 +-
 misp_modules/modules/expansion/__init__.py    |  4 +-
 .../modules/expansion/_vmray/__init__.py      |  0
 .../_vmray}/vmray_rest_api.py                 |  8 +-
 .../modules/expansion/vmray_submit.py         | 27 +++---
 misp_modules/modules/import_mod/__init__.py   |  4 +-
 .../modules/import_mod/_vmray/__init__.py     |  0
 .../_vmray}/vmray_rest_api.py                 |  6 +-
 .../modules/import_mod/vmray_import.py        | 97 +++++++++----------
 9 files changed, 67 insertions(+), 83 deletions(-)
 create mode 100644 misp_modules/modules/expansion/_vmray/__init__.py
 rename misp_modules/modules/{import_mod => expansion/_vmray}/vmray_rest_api.py (97%)
 create mode 100644 misp_modules/modules/import_mod/_vmray/__init__.py
 rename misp_modules/modules/{expansion => import_mod/_vmray}/vmray_rest_api.py (97%)

diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py
index a2562c4..1edfc86 100644
--- a/misp_modules/__init__.py
+++ b/misp_modules/__init__.py
@@ -123,6 +123,8 @@ def load_modules(mod_dir):
         if os.path.basename(root).startswith("."):
             continue
         for filename in fnmatch.filter(filenames, '*.py'):
+            if root.split('/')[-1].startswith('_'):
+                continue
             if filename == '__init__.py':
                 continue
             modulename = filename.split(".")[0]
@@ -145,7 +147,7 @@ def load_package_modules():
     mhandlers = {}
     modules = []
     for path, module in sys.modules.items():
-        r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
+        r = re.findall("misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
         if r and len(r[0]) == 2:
             moduletype, modulename = r[0]
             mhandlers[modulename] = module
diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py
index fdbdfb8..62a6ecd 100644
--- a/misp_modules/modules/expansion/__init__.py
+++ b/misp_modules/modules/expansion/__init__.py
@@ -1,2 +1,4 @@
-__all__ = ['vmray_submit','asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
+from . import _vmray
+
+__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
            'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'wiki']
diff --git a/misp_modules/modules/expansion/_vmray/__init__.py b/misp_modules/modules/expansion/_vmray/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/misp_modules/modules/import_mod/vmray_rest_api.py b/misp_modules/modules/expansion/_vmray/vmray_rest_api.py
similarity index 97%
rename from misp_modules/modules/import_mod/vmray_rest_api.py
rename to misp_modules/modules/expansion/_vmray/vmray_rest_api.py
index 64b8024..4d5245b 100644
--- a/misp_modules/modules/import_mod/vmray_rest_api.py
+++ b/misp_modules/modules/expansion/_vmray/vmray_rest_api.py
@@ -1,16 +1,12 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 """Python client library for VMRay REST API"""
 
 import base64
 import datetime
 import os.path
 import requests
-#import urlparse
 import urllib.parse
 
-from io import IOBase
-from io import BytesIO
-
 # disable nasty certification warning
 # pylint: disable=no-member
 try:
@@ -87,7 +83,7 @@ class VMRayRESTAPI(object):
                     filename = value["filename"]
                     sample = value["data"]
                     file_params[key] = (filename, sample, "application/octet-stream")
-                elif isinstance(value, file) or hasattr(value, "read"):
+                elif hasattr(value, "read"):
                     filename = os.path.split(value.name)[1]
                     # For the following block refer to DEV-1820
                     try:
diff --git a/misp_modules/modules/expansion/vmray_submit.py b/misp_modules/modules/expansion/vmray_submit.py
index 9ab4e05..407853c 100644
--- a/misp_modules/modules/expansion/vmray_submit.py
+++ b/misp_modules/modules/expansion/vmray_submit.py
@@ -12,16 +12,12 @@ TODO:
 '''
 
 import json
-import re
 import base64
 
-import sys
-import os
-base_dir = os.path.dirname(__file__) or '.'
-sys.path.append(base_dir)
-from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError
 import io
 
+from ._vmray.vmray_rest_api import VMRayRESTAPI
+
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['attachment'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']}
 moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe',
@@ -81,7 +77,7 @@ def handler(q=False):
     if data and attachment:
         args = {}
         args["shareable"] = shareable
-        args["sample_file"] = {'data': io.BytesIO( data ) , 'filename': attachment }
+        args["sample_file"] = {'data': io.BytesIO(data), 'filename': attachment}
         args["reanalyze"] = reanalyze
 
         try:
@@ -118,13 +114,13 @@ def vmrayProcess(vmraydata):
             # Result received?
             if submissions and jobs:
                 r = {'results': []}
-                r["results"].append( {"types": "md5", "values": submissions["submission_sample_md5"]} )
-                r["results"].append( {"types": "sha1", "values": submissions["submission_sample_sha1"]} )
-                r["results"].append( {"types": "sha256", "values": submissions["submission_sample_sha256"]} )
-                r["results"].append( {"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]} )
-                r["results"].append( {"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]} )
-                r["results"].append( {"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]} )
-                r["results"].append( {"types": "link", "values": submissions["submission_webif_url"]} )
+                r["results"].append({"types": "md5", "values": submissions["submission_sample_md5"]})
+                r["results"].append({"types": "sha1", "values": submissions["submission_sample_sha1"]})
+                r["results"].append({"types": "sha256", "values": submissions["submission_sample_sha256"]})
+                r["results"].append({"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]})
+                r["results"].append({"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]})
+                r["results"].append({"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]})
+                r["results"].append({"types": "link", "values": submissions["submission_webif_url"]})
 
                 # Include data from different jobs
                 if include_vmrayjobids:
@@ -132,7 +128,7 @@ def vmrayProcess(vmraydata):
                         job_id = job["job_id"]
                         job_vm_name = job["job_vm_name"]
                         job_configuration_name = job["job_configuration_name"]
-                        r["results"].append( {"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name) })
+                        r["results"].append({"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name)})
                 return r
             else:
                 misperrors['error'] = "No valid results returned."
@@ -149,4 +145,3 @@ def vmraySubmit(api, args):
     ''' Submit the sample to VMRay'''
     vmraydata = api.call("POST", "/rest/sample/submit", args)
     return vmraydata
-
diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py
index 4b30a1a..70b674a 100644
--- a/misp_modules/modules/import_mod/__init__.py
+++ b/misp_modules/modules/import_mod/__init__.py
@@ -1 +1,3 @@
-__all__ = ['vmray_import','testimport', 'ocr', 'stiximport']
+from . import _vmray
+
+__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport']
diff --git a/misp_modules/modules/import_mod/_vmray/__init__.py b/misp_modules/modules/import_mod/_vmray/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/misp_modules/modules/expansion/vmray_rest_api.py b/misp_modules/modules/import_mod/_vmray/vmray_rest_api.py
similarity index 97%
rename from misp_modules/modules/expansion/vmray_rest_api.py
rename to misp_modules/modules/import_mod/_vmray/vmray_rest_api.py
index 64b8024..d37c6f2 100644
--- a/misp_modules/modules/expansion/vmray_rest_api.py
+++ b/misp_modules/modules/import_mod/_vmray/vmray_rest_api.py
@@ -5,12 +5,8 @@ import base64
 import datetime
 import os.path
 import requests
-#import urlparse
 import urllib.parse
 
-from io import IOBase
-from io import BytesIO
-
 # disable nasty certification warning
 # pylint: disable=no-member
 try:
@@ -87,7 +83,7 @@ class VMRayRESTAPI(object):
                     filename = value["filename"]
                     sample = value["data"]
                     file_params[key] = (filename, sample, "application/octet-stream")
-                elif isinstance(value, file) or hasattr(value, "read"):
+                elif hasattr(value, "read"):
                     filename = os.path.split(value.name)[1]
                     # For the following block refer to DEV-1820
                     try:
diff --git a/misp_modules/modules/import_mod/vmray_import.py b/misp_modules/modules/import_mod/vmray_import.py
index 397bd39..dceeb1f 100644
--- a/misp_modules/modules/import_mod/vmray_import.py
+++ b/misp_modules/modules/import_mod/vmray_import.py
@@ -16,42 +16,36 @@ TODO:
 
 import json
 import re
-import sys
-import os
-base_dir = os.path.dirname(__file__) or '.'
-sys.path.append(base_dir)
-from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError
+
+from ._vmray.vmray_rest_api import VMRayRESTAPI
 
 misperrors = {'error': 'Error'}
 inputSource = []
 moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe',
               'description': 'Import VMRay (VTI) results',
               'module-type': ['import']}
-userConfig = {
-               'include_textdescr': {
-                 'type': 'Boolean',
-                 'message': 'Include textual description'
-               },
-               'include_analysisid': {
-                 'type': 'Boolean',
-                 'message': 'Include VMRay analysis_id text'
-               },
-               'only_network_info': {
-                 'type': 'Boolean',
-                 'message': 'Only include network (src-ip, hostname, domain, ...) information'
-               },
-               'sample_id': {
-                 'type': 'Integer',
-                 'errorMessage': 'Expected a sample ID',
-                 'message': 'The VMRay sample_id'
-               }
-             };
+userConfig = {'include_textdescr': {'type': 'Boolean',
+                                    'message': 'Include textual description'
+                                    },
+              'include_analysisid': {'type': 'Boolean',
+                                     'message': 'Include VMRay analysis_id text'
+                                     },
+              'only_network_info': {'type': 'Boolean',
+                                    'message': 'Only include network (src-ip, hostname, domain, ...) information'
+                                    },
+              'sample_id': {'type': 'Integer',
+                            'errorMessage': 'Expected a sample ID',
+                            'message': 'The VMRay sample_id'
+                            }
+              }
+
 moduleconfig = ['apikey', 'url']
 
 include_textdescr = False
 include_analysisid = False
 only_network_info = False
 
+
 def handler(q=False):
     global include_textdescr
     global include_analysisid
@@ -104,8 +98,8 @@ def handler(q=False):
                                     url1 = "https://cloud.vmray.com/user/analysis/view?from_sample_id=%u" % sample_id
                                     url2 = "&id=%u" % analysis_id
                                     url3 = "&sub=%2Freport%2Foverview.html"
-                                    p["results"].append({ "values": url1 + url2 + url3, "types": "link" })
-                                vmray_results = {'results': vmray_results["results"] + p["results"] }
+                                    p["results"].append({"values": url1 + url2 + url3, "types": "link"})
+                                vmray_results = {'results': vmray_results["results"] + p["results"]}
 
                 # Clean up (remove doubles)
                 vmray_results = vmrayCleanup(vmray_results)
@@ -121,7 +115,6 @@ def handler(q=False):
         return misperrors
 
 
-
 def introspection():
     modulesetup = {}
     try:
@@ -175,42 +168,42 @@ def vmrayVtiPatterns(vti_patterns):
             elif pattern["category"] == "_network" and pattern["operation"] == "_connect":
                 content = vmrayConnect(pattern)
 
-            elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page":
+            elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page":
                 content = vmrayGeneric(pattern)
-            elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint":
+            elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint":
                 content = vmrayGeneric(pattern, "mutex", 1)
 
-            elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
+            elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
                 content = vmrayGeneric(pattern)
-            elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage":
+            elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage":
                 content = vmrayGeneric(pattern)
 
-            elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file":
+            elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file":
                 content = vmrayGeneric(pattern, "filename", 1)
-            elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file":
+            elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file":
                 content = vmrayGeneric(pattern, "filename", 1)
 
-            elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory":
+            elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory":
                 content = vmrayGeneric(pattern)
-            elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow":
+            elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow":
                 content = vmrayGeneric(pattern)
-            elif only_network_info == False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files":
+            elif only_network_info is False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files":
                 content = vmrayGeneric(pattern)
 
-            elif only_network_info == False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script":
+            elif only_network_info is False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script":
                 content = vmrayGeneric(pattern, "regkey", 1)
-            elif only_network_info == False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges":
+            elif only_network_info is False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges":
                 content = vmrayGeneric(pattern)
 
             if content:
-                r["results"].append( content["attributes"] )
-                r["results"].append( content["text"] )
+                r["results"].append(content["attributes"])
+                r["results"].append(content["text"])
 
         # Remove empty results
-        r["results"] = [x for x in r["results"] if isinstance(x, dict) and  len(x["values"]) != 0]
+        r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0]
         for el in r["results"]:
-            if not el in y["results"]:
-                y["results"].append( el )
+            if el not in y["results"]:
+                y["results"].append(el)
         return y
     else:
         return False
@@ -221,22 +214,22 @@ def vmrayCleanup(x):
     y = {'results': []}
 
     for el in x["results"]:
-        if not el in y["results"]:
-            y["results"].append( el )
+        if el not in y["results"]:
+            y["results"].append(el)
     return y
 
 
 def vmraySanitizeInput(s):
     ''' Sanitize some input so it gets properly imported in MISP'''
     if s:
-        s = s.replace('"','')
-        s =  re.sub('\\\\', r'\\', s)
+        s = s.replace('"', '')
+        s = re.sub('\\\\', r'\\', s)
         return s
     else:
         return False
 
 
-def vmrayGeneric(el, attr = "", attrpos = 1):
+def vmrayGeneric(el, attr="", attrpos=1):
     ''' Convert a 'generic' VTI pattern to MISP data'''
 
     r = {"values": []}
@@ -257,8 +250,7 @@ def vmrayGeneric(el, attr = "", attrpos = 1):
                 f["values"].append(vmraySanitizeInput(content))
                 f["types"] = ["text"]
 
-            return {    "text": f,
-                        "attributes": r}
+            return {"text": f, "attributes": r}
         else:
             return False
     else:
@@ -276,7 +268,7 @@ def vmrayConnect(el):
         content = el["technique_desc"]
         if content:
             target = content.split("\"")
-            port = (target[1].split(":"))[1]
+            # port = (target[1].split(":"))[1]  ## FIXME: not used
             host = (target[1].split(":"))[0]
             if ipre.match(str(host)):
                 r["values"].append(host)
@@ -292,8 +284,7 @@ def vmrayConnect(el):
                 f["values"].append(vmraySanitizeInput(content))
                 f["types"] = ["text"]
 
-            return {    "text": f,
-                        "attributes": r}
+            return {"text": f, "attributes": r}
         else:
             return False
     else:

From 5624104b7770a4849a4f44bf58ce0600c7628d52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Tue, 15 Nov 2016 16:47:17 +0100
Subject: [PATCH 07/54] Fix STIX import module

---
 misp_modules/modules/import_mod/stiximport.py |  9 ++++---
 tests/test.py                                 | 24 ++++++++++---------
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/misp_modules/modules/import_mod/stiximport.py b/misp_modules/modules/import_mod/stiximport.py
index 0da0372..16a2f22 100755
--- a/misp_modules/modules/import_mod/stiximport.py
+++ b/misp_modules/modules/import_mod/stiximport.py
@@ -28,7 +28,7 @@ def handler(q=False):
     q = json.loads(q)
 
     # It's b64 encoded, so decode that stuff
-    package = str(base64.b64decode(q.get("data", None)), 'utf-8')
+    package = base64.b64decode(q.get("data")).decode('utf-8')
 
     # If something really weird happened
     if not package:
@@ -168,6 +168,9 @@ def buildObservable(o):
         # May as well be useless
         return r
 
+    if not o.get('object'):
+        return r
+
     props = o["object"]["properties"]
 
     # If it has an address_value field, it's gonna be an address
@@ -195,7 +198,7 @@ def buildObservable(o):
             for hsh in props["hashes"]:
                 r["values"].append(hsh["simple_hash_value"]["value"])
                 r["types"] = identifyHash(hsh["simple_hash_value"]["value"])
-        
+
     elif "xsi:type" in props:
       # Cybox. Ew.
       try:
@@ -208,7 +211,7 @@ def buildObservable(o):
         else:
           print("Ignoring {}".format(type_))
       except:
-        pass 
+        pass
     return r
 
 
diff --git a/tests/test.py b/tests/test.py
index 2f000ec..d506595 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 import unittest
@@ -7,6 +7,7 @@ import base64
 import json
 import os
 
+
 class TestModules(unittest.TestCase):
 
     def setUp(self):
@@ -32,13 +33,14 @@ class TestModules(unittest.TestCase):
             print(response.json())
 
     def test_stix(self):
-        with open("tests/stix.xml", "r") as f:
-            data = json.dumps({"module":"stiximport",
-                    "data":str(base64.b64encode(bytes(f.read(), 'utf-8')), 'utf-8'),
-                    "config": {"max_size": "15000"},
-                   })
+        with open("tests/stix.xml", "rb") as f:
+            content = base64.b64encode(f.read())
+            data = json.dumps({"module": "stiximport",
+                               "data": content.decode('utf-8'),
+                               "config": {"max_size": "15000"},
+                               })
             response = requests.post(self.url + "query", data=data)
-            print(response.json())
+            print('STIX', response.json())
 
     def test_virustotal(self):
         # This can't actually be tested without disclosing a private
@@ -46,11 +48,11 @@ class TestModules(unittest.TestCase):
         # and pass if it can't find one
 
         if not os.path.exists("tests/bodyvirustotal.json"):
-          return
-        
+            return
+
         with open("tests/bodyvirustotal.json", "r") as f:
-          response = requests.post(self.url + "query", data=f.read()).json()
+            response = requests.post(self.url + "query", data=f.read()).json()
         assert(response)
 
 if __name__ == '__main__':
-  unittest.main()
+    unittest.main()

From 3253d92b421981141152f6858201e319b213d16d Mon Sep 17 00:00:00 2001
From: Koen Van Impe <koen.vanimpe@cudeso.be>
Date: Fri, 18 Nov 2016 18:23:52 +0100
Subject: [PATCH 08/54] Submit malware samples

_submit now includes malware samples (zipped content from misp)
_import checks when no vti_results are returned + bugfix
---
 .../modules/expansion/vmray_submit.py         | 29 +++++++++++----
 .../modules/import_mod/vmray_import.py        | 35 ++++++++++++-------
 2 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/misp_modules/modules/expansion/vmray_submit.py b/misp_modules/modules/expansion/vmray_submit.py
index 407853c..15b163b 100644
--- a/misp_modules/modules/expansion/vmray_submit.py
+++ b/misp_modules/modules/expansion/vmray_submit.py
@@ -6,7 +6,6 @@ Submit sample to  VMRay.
 Submit a sample to VMRay
 
 TODO:
- # Deal with malicious samples (ZIP file, 'infected')
  # Deal with archive submissions
 
 '''
@@ -15,12 +14,13 @@ import json
 import base64
 
 import io
+import zipfile
 
 from ._vmray.vmray_rest_api import VMRayRESTAPI
 
 misperrors = {'error': 'Error'}
-mispattributes = {'input': ['attachment'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']}
-moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe',
+mispattributes = {'input': ['attachment', 'malware-sample'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']}
+moduleinfo = {'version': '0.2', 'author': 'Koen Van Impe',
               'description': 'Submit a sample to VMRay',
               'module-type': ['expansion']}
 moduleconfig = ['apikey', 'url', 'shareable', 'do_not_reanalyze', 'do_not_include_vmrayjobids']
@@ -38,8 +38,23 @@ def handler(q=False):
 
     try:
         data = request.get("data")
-        attachment = request.get("attachment")
-        data = base64.b64decode(data)
+        if 'malware-sample' in request:
+            # malicious samples are encrypted with zip (password infected) and then base64 encoded
+            sample_filename = request.get("malware-sample").split("|",1)[0]
+            data = base64.b64decode(data)
+            fl = io.BytesIO(data)
+            zf = zipfile.ZipFile(fl)
+            sample_hashname = zf.namelist()[0]
+            data = zf.read(sample_hashname,b"infected")
+            zf.close()
+        elif 'attachment' in request:
+            # All attachments get base64 encoded
+            sample_filename = request.get("attachment")
+            data = base64.b64decode(data)
+
+        else:
+            misperrors['error'] = "No malware sample or attachment supplied"
+            return misperrors
     except:
         misperrors['error'] = "Unable to process submited sample data"
         return misperrors
@@ -74,10 +89,10 @@ def handler(q=False):
         do_not_include_vmrayjobids = False
     include_vmrayjobids = not do_not_include_vmrayjobids
 
-    if data and attachment:
+    if data and sample_filename:
         args = {}
         args["shareable"] = shareable
-        args["sample_file"] = {'data': io.BytesIO(data), 'filename': attachment}
+        args["sample_file"] = {'data': io.BytesIO(data), 'filename': sample_filename}
         args["reanalyze"] = reanalyze
 
         try:
diff --git a/misp_modules/modules/import_mod/vmray_import.py b/misp_modules/modules/import_mod/vmray_import.py
index dceeb1f..a7b7110 100644
--- a/misp_modules/modules/import_mod/vmray_import.py
+++ b/misp_modules/modules/import_mod/vmray_import.py
@@ -84,6 +84,7 @@ def handler(q=False):
             # Get all information on the sample, returns a set of finished analyze jobs
             data = vmrayGetInfoAnalysis(api, sample_id)
             if data["data"]:
+                vti_patterns_found = False
                 for analysis in data["data"]:
                     analysis_id = analysis["analysis_id"]
 
@@ -93,17 +94,23 @@ def handler(q=False):
 
                         if analysis_data:
                             p = vmrayVtiPatterns(analysis_data["vti_patterns"])
-                            if p:
-                                if include_analysisid:
-                                    url1 = "https://cloud.vmray.com/user/analysis/view?from_sample_id=%u" % sample_id
-                                    url2 = "&id=%u" % analysis_id
-                                    url3 = "&sub=%2Freport%2Foverview.html"
-                                    p["results"].append({"values": url1 + url2 + url3, "types": "link"})
+                            if p and len(p["results"]) > 0:
+                                vti_patterns_found = True
                                 vmray_results = {'results': vmray_results["results"] + p["results"]}
-
+                            if include_analysisid:
+                                a_id = {'results': []}
+                                url1 = "https://cloud.vmray.com/user/analysis/view?from_sample_id=%u" % sample_id
+                                url2 = "&id=%u" % analysis_id
+                                url3 = "&sub=%2Freport%2Foverview.html"
+                                a_id["results"].append({ "values": url1 + url2 + url3, "types": "link" })
+                                vmray_results = {'results': vmray_results["results"] + a_id["results"] }
                 # Clean up (remove doubles)
-                vmray_results = vmrayCleanup(vmray_results)
-                return vmray_results
+                if vti_patterns_found:
+                    vmray_results = vmrayCleanup(vmray_results)
+                    return vmray_results
+                else:
+                    misperrors['error'] = "No vti_results returned or jobs not finished"
+                    return misperrors
             else:
                 misperrors['error'] = "Unable to fetch sample id %u" % (sample_id)
                 return misperrors
@@ -172,6 +179,8 @@ def vmrayVtiPatterns(vti_patterns):
                 content = vmrayGeneric(pattern)
             elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint":
                 content = vmrayGeneric(pattern, "mutex", 1)
+            elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_crashed_process":
+                content = vmrayGeneric(pattern)
 
             elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
                 content = vmrayGeneric(pattern)
@@ -240,9 +249,11 @@ def vmrayGeneric(el, attr="", attrpos=1):
         if content:
             if attr:
                 content_split = content.split("\"")
-                content_split[attrpos] = vmraySanitizeInput(content_split[attrpos])
-                r["values"].append(content_split[attrpos])
-                r["types"] = [attr]
+                # Attributes are between open " and close "; so use >
+                if len(content_split) > attrpos:
+                    content_split[attrpos] = vmraySanitizeInput(content_split[attrpos])
+                    r["values"].append(content_split[attrpos])
+                    r["types"] = [attr]                
 
             # Adding the value also as text to get the extra description,
             # but this is pretty useless for "url"

From c567d1e6f28e7fe3c9dcd2da0ae10cc89deb821b Mon Sep 17 00:00:00 2001
From: Hannah Ward <Hannah.ward9001@gmail.com>
Date: Mon, 21 Nov 2016 10:59:30 +0000
Subject: [PATCH 09/54] Moved to misp_stix_converter

---
 misp_modules/modules/import_mod/stiximport.py | 215 +-----------------
 setup.py                                      |   3 +-
 tests/test.py                                 |  11 +-
 3 files changed, 22 insertions(+), 207 deletions(-)

diff --git a/misp_modules/modules/import_mod/stiximport.py b/misp_modules/modules/import_mod/stiximport.py
index 16a2f22..99e891c 100755
--- a/misp_modules/modules/import_mod/stiximport.py
+++ b/misp_modules/modules/import_mod/stiximport.py
@@ -1,19 +1,21 @@
 import json
-from stix.core import STIXPackage
 import re
 import base64
 import hashlib
 import tempfile
+import os
+
+from pymisp.tools import stix
 
 misperrors = {'error': 'Error'}
 userConfig = {}
 inputSource = ['file']
 
-moduleinfo = {'version': '0.1', 'author': 'Hannah Ward',
+moduleinfo = {'version': '0.2', 'author': 'Hannah Ward',
               'description': 'Import some stix stuff',
               'module-type': ['import']}
 
-moduleconfig = ["max_size"]
+moduleconfig = []
 
 
 def handler(q=False):
@@ -34,211 +36,18 @@ def handler(q=False):
     if not package:
         return json.dumps({"success": 0})
 
-    # Get the maxsize from the config
-    # Default to 10MB
-    # (I believe the max_size arg is given  in bytes)
-    # Check if we were given a configuration
-    memsize = q.get("config", None)
+    tfile = tempfile.NamedTemporaryFile(mode="w", prefix="STIX", delete=False)
+    tfile.write(package)
+    tfile.close()
 
-    # If we were, find out if there's a memsize field
-    if memsize:
-        memsize = memsize.get("max_size", 10 * 1024)
-    else:
-        memsize = 10 * 1024
+    pkg = stix.load_stix(tfile.name)
 
-    # Load up the package into STIX
-    package = loadPackage(package, memsize)
+    for attrib in pkg.attributes:
+        r["results"].append({ "values" : [attrib.value] , "types": [attrib.type], "categories": [attrib.category]})
 
-    # Build all the observables
-    if package.observables:
-        for obs in package.observables:
-            r["results"].append(buildObservable(obs))
-
-    # And now the threat actors
-    if package.threat_actors:
-        for ta in package.threat_actors:
-            r["results"].append(buildActor(ta))
-
-    # Aaaand the indicators
-    if package.indicators:
-        for ind in package.indicators:
-            r["results"] += buildIndicator(ind)
-
-    # Are you seeing a pattern?
-    if package.exploit_targets:
-        for et in package.exploit_targets:
-            r["results"].append(buildExploitTarget(et))
-
-    # LOADING STUFF
-    if package.campaigns:
-        for cpn in package.campaigns:
-            r["results"].append(buildCampaign(cpn))
-
-    # Clean up results
-    # Don't send on anything that didn't have a value
-    r["results"] = [x for x in r["results"] if isinstance(x, dict) and  len(x["values"]) != 0]
+    os.unlink(tfile.name)
     return r
 
-# Quick and dirty regex for IP addresses
-ipre = re.compile("([0-9]{1,3}.){3}[0-9]{1,3}")
-
-
-def buildCampaign(cpn):
-    """
-        Extract a campaign name
-    """
-    return {"values": [cpn.title], "types": ["campaign-name"]}
-
-
-def buildExploitTarget(et):
-    """
-        Extract CVEs from exploit targets
-    """
-
-    r = {"values": [], "types": ["vulnerability"]}
-
-    if et.vulnerabilities:
-        for v in et.vulnerabilities:
-            if v.cve_id:
-                r["values"].append(v.cve_id)
-    return r
-
-
-
-def identifyHash(hsh):
-    """
-        What's that hash!?
-    """
-
-    possible_hashes = []
-
-    hashes = [x for x in hashlib.algorithms_guaranteed]
-
-    for h in hashes:
-        if len(str(hsh)) == len(hashlib.new(h).hexdigest()):
-            possible_hashes.append(h)
-            possible_hashes.append("filename|{}".format(h))
-    return possible_hashes
-
-
-def buildIndicator(ind):
-    """
-        Extract hashes
-        and other fun things
-        like that
-    """
-    r = []
-    # Try to get hashes. I hate stix
-    if ind.observables:
-        for i in ind.observables:
-          if i.observable_composition:
-            for j in i.observable_composition.observables:
-              r.append(buildObservable(j))
-          r.append(buildObservable(i))
-    return r
-
-
-def buildActor(ta):
-    """
-        Extract the name
-        and comment of a
-        threat actor
-    """
-
-    r = {"values": [ta.title], "types": ["threat-actor"]}
-
-    return r
-
-
-def buildObservable(o):
-    """
-        Take a STIX observable
-        and extract the value
-        and category
-    """
-    # Life is easier with json
-    if not isinstance(o, dict):
-        o = json.loads(o.to_json())
-    # Make a new record to store values in
-    r = {"values": []}
-
-    # Get the object properties. This contains all the
-    # fun stuff like values
-    if "observable_composition" in o:
-        # May as well be useless
-        return r
-
-    if not o.get('object'):
-        return r
-
-    props = o["object"]["properties"]
-
-    # If it has an address_value field, it's gonna be an address
-    # Kinda obvious really
-    if "address_value" in props:
-
-        # We've got ourselves a nice little address
-        value = props["address_value"]
-
-        if isinstance(value, dict):
-            # Sometimes it's embedded in a dictionary
-            value = value["value"]
-
-        # Is it an IP?
-        if ipre.match(str(value)):
-            # Yes!
-            r["values"].append(value)
-            r["types"] = ["ip-src", "ip-dst"]
-        else:
-            # Probably a domain yo
-            r["values"].append(value)
-            r["types"] = ["domain", "hostname"]
-
-        if "hashes" in props:
-            for hsh in props["hashes"]:
-                r["values"].append(hsh["simple_hash_value"]["value"])
-                r["types"] = identifyHash(hsh["simple_hash_value"]["value"])
-
-    elif "xsi:type" in props:
-      # Cybox. Ew.
-      try:
-        type_ = props["xsi:type"]
-        val   = props["value"]
-
-        if type_ == "LinkObjectType":
-          r["types"] = ["link"]
-          r["values"].append(val)
-        else:
-          print("Ignoring {}".format(type_))
-      except:
-        pass
-    return r
-
-
-def loadPackage(data, memsize=1024):
-    # Write the stix package to a tmp file
-
-    temp = tempfile.SpooledTemporaryFile(max_size=int(memsize), mode="w+")
-
-    temp.write(data)
-
-    # Back to the beginning so we can read it again
-    temp.seek(0)
-    try:
-        # Try loading it into every format we know of
-        try:
-            package = STIXPackage().from_xml(temp)
-        except:
-            # We have to seek back again
-            temp.seek(0)
-            package = STIXPackage().from_json(temp)
-    except Exception:
-        print("Failed to load package")
-        raise ValueError("COULD NOT LOAD STIX PACKAGE!")
-    temp.close()
-    return package
-
-
 def introspection():
     modulesetup = {}
     try:
diff --git a/setup.py b/setup.py
index 8ad517e..0010c05 100644
--- a/setup.py
+++ b/setup.py
@@ -38,5 +38,6 @@ setup(
         'pillow',
         'pytesseract',
         'shodan',
-    ]
+        'misp_stix_converter'
+    ],
 )
diff --git a/tests/test.py b/tests/test.py
index d506595..faf5173 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -37,10 +37,15 @@ class TestModules(unittest.TestCase):
             content = base64.b64encode(f.read())
             data = json.dumps({"module": "stiximport",
                                "data": content.decode('utf-8'),
-                               "config": {"max_size": "15000"},
                                })
-            response = requests.post(self.url + "query", data=data)
-            print('STIX', response.json())
+            response = requests.post(self.url + "query", data=data).json()
+
+            print("STIX :: {}".format(response))
+            values = [x["values"][0] for x in response["results"]]
+
+            assert("209.239.79.47" in values)
+            assert("41.213.121.180" in values)
+            assert("eu-society.com" in values)
 
     def test_virustotal(self):
         # This can't actually be tested without disclosing a private

From b35c0ae4de4d2044e7800731018b3201ca90eb87 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Mon, 21 Nov 2016 12:09:44 +0100
Subject: [PATCH 10/54] VMRay import module added

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index fbc9400..b523140 100644
--- a/README.md
+++ b/README.md
@@ -37,6 +37,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 
 * [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes.
 * [stiximport](misp_modules/modules/import_mod/stiximport.py) - An import module to process STIX xml/json
+* [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export
 
 ## How to install and start MISP modules?
 

From 0dfea440014b4e1701fd42a20c45f4d8992c00bb Mon Sep 17 00:00:00 2001
From: Hannah Ward <Hannah.ward9001@gmail.com>
Date: Mon, 21 Nov 2016 11:57:04 +0000
Subject: [PATCH 11/54] Use SpooledTemp, not NamedTemp file

---
 misp_modules/modules/import_mod/stiximport.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/misp_modules/modules/import_mod/stiximport.py b/misp_modules/modules/import_mod/stiximport.py
index 99e891c..4905cd2 100755
--- a/misp_modules/modules/import_mod/stiximport.py
+++ b/misp_modules/modules/import_mod/stiximport.py
@@ -2,8 +2,6 @@ import json
 import re
 import base64
 import hashlib
-import tempfile
-import os
 
 from pymisp.tools import stix
 
@@ -36,16 +34,11 @@ def handler(q=False):
     if not package:
         return json.dumps({"success": 0})
 
-    tfile = tempfile.NamedTemporaryFile(mode="w", prefix="STIX", delete=False)
-    tfile.write(package)
-    tfile.close()
-
-    pkg = stix.load_stix(tfile.name)
+    pkg = stix.load_stix(package)
 
     for attrib in pkg.attributes:
         r["results"].append({ "values" : [attrib.value] , "types": [attrib.type], "categories": [attrib.category]})
 
-    os.unlink(tfile.name)
     return r
 
 def introspection():

From 1f49f362050aeaeefc4b562c50d327406fd26704 Mon Sep 17 00:00:00 2001
From: Hannah Ward <Hannah.ward9001@gmail.com>
Date: Mon, 21 Nov 2016 13:05:07 +0000
Subject: [PATCH 12/54] Removed unneeded modules

---
 misp_modules/modules/import_mod/stiximport.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/misp_modules/modules/import_mod/stiximport.py b/misp_modules/modules/import_mod/stiximport.py
index 4905cd2..39ac9dc 100755
--- a/misp_modules/modules/import_mod/stiximport.py
+++ b/misp_modules/modules/import_mod/stiximport.py
@@ -1,7 +1,5 @@
 import json
-import re
 import base64
-import hashlib
 
 from pymisp.tools import stix
 

From 454c450abd4ca75ee0bb66246354558478e5ab19 Mon Sep 17 00:00:00 2001
From: Hannah Ward <Hannah.ward9001@gmail.com>
Date: Mon, 21 Nov 2016 13:13:24 +0000
Subject: [PATCH 13/54] Don't cache anything pls travis

---
 .travis.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index fbe3487..5653c1e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,5 @@
 language: python
 
-cache: pip
-
 services:
     - redis-server
 

From 35538b4e87f711df5fd6ed59dca4a5e36cc15f0c Mon Sep 17 00:00:00 2001
From: Hannah Ward <Hannah.ward9001@gmail.com>
Date: Mon, 21 Nov 2016 14:08:20 +0000
Subject: [PATCH 14/54] Added pymisp to reqs

---
 REQUIREMENTS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/REQUIREMENTS b/REQUIREMENTS
index 92a84d7..6bb1d47 100644
--- a/REQUIREMENTS
+++ b/REQUIREMENTS
@@ -12,6 +12,7 @@ pyeupi
 ipasn-redis
 asnhistory
 git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client
+git+https://github.com/MISP/PyMISP.git#egg=pymisp
 pillow
 pytesseract
 SPARQLWrapper

From eb538810acee730f8b29f54242e010ea04682610 Mon Sep 17 00:00:00 2001
From: Hannah Ward <Hannah.ward9001@gmail.com>
Date: Mon, 21 Nov 2016 14:31:48 +0000
Subject: [PATCH 15/54] Maybe it'll take the git repo now?

---
 REQUIREMENTS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/REQUIREMENTS b/REQUIREMENTS
index 6bb1d47..23e1cbe 100644
--- a/REQUIREMENTS
+++ b/REQUIREMENTS
@@ -12,7 +12,7 @@ pyeupi
 ipasn-redis
 asnhistory
 git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client
-git+https://github.com/MISP/PyMISP.git#egg=pymisp
+git+https://github.com/MISP/PyMISP.git
 pillow
 pytesseract
 SPARQLWrapper

From 5372f5029891124a0a8a806bd724f3b658c13684 Mon Sep 17 00:00:00 2001
From: Hannah Ward <Hannah.ward9001@gmail.com>
Date: Mon, 21 Nov 2016 14:43:29 +0000
Subject: [PATCH 16/54] Travis should now use the master branch

---
 .travis.yml  | 6 +++++-
 REQUIREMENTS | 1 -
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 5653c1e..3462f08 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,11 @@ install:
     - pip install codecov
     - pip install -U -r REQUIREMENTS
     - pip install .
-
+    # Force travis to use the right pymisp
+    - git clone https://github.com/MISP/PyMISP.git
+    - cd PyMISP 
+    - python setup.py install
+    - cd ..
 script:
     - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ &
     - pid=$!
diff --git a/REQUIREMENTS b/REQUIREMENTS
index 23e1cbe..92a84d7 100644
--- a/REQUIREMENTS
+++ b/REQUIREMENTS
@@ -12,7 +12,6 @@ pyeupi
 ipasn-redis
 asnhistory
 git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client
-git+https://github.com/MISP/PyMISP.git
 pillow
 pytesseract
 SPARQLWrapper

From d60f4ef4958ad67abe401175b09fa5f65bb9f1d4 Mon Sep 17 00:00:00 2001
From: Hannah Ward <Hannah.ward9001@gmail.com>
Date: Mon, 21 Nov 2016 14:49:05 +0000
Subject: [PATCH 17/54] Use the CIRCL pymisp. Silly @rafiot ;)

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 3462f08..15858f4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,7 +17,7 @@ install:
     - pip install -U -r REQUIREMENTS
     - pip install .
     # Force travis to use the right pymisp
-    - git clone https://github.com/MISP/PyMISP.git
+    - git clone https://github.com/CIRCL/PyMISP.git
     - cd PyMISP 
     - python setup.py install
     - cd ..

From 22f4edd25d541e769d8320210c2ec5a6e5227c2e Mon Sep 17 00:00:00 2001
From: Hannah Ward <Hannah.ward9001@gmail.com>
Date: Mon, 21 Nov 2016 15:07:56 +0000
Subject: [PATCH 18/54] Ok we'll use the dep from misp-stix-converter. Surely
 this'll work?

---
 .travis.yml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 15858f4..5653c1e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,11 +16,7 @@ install:
     - pip install codecov
     - pip install -U -r REQUIREMENTS
     - pip install .
-    # Force travis to use the right pymisp
-    - git clone https://github.com/CIRCL/PyMISP.git
-    - cd PyMISP 
-    - python setup.py install
-    - cd ..
+
 script:
     - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ &
     - pid=$!

From daa66dd32f13231a5d9de54c4508051187fcdda3 Mon Sep 17 00:00:00 2001
From: Hannah Ward <Hannah.ward9001@gmail.com>
Date: Mon, 21 Nov 2016 15:20:57 +0000
Subject: [PATCH 19/54] Use git for everything we can

---
 REQUIREMENTS | 2 ++
 setup.py     | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/REQUIREMENTS b/REQUIREMENTS
index 92a84d7..16f5512 100644
--- a/REQUIREMENTS
+++ b/REQUIREMENTS
@@ -12,6 +12,8 @@ pyeupi
 ipasn-redis
 asnhistory
 git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client
+git+https://github.com/MISP/MISP-STIX-Converter.git#egg=misp_stix_converter
+git+https://github.com/CIRCL/PyMISP.git#egg=pymisp
 pillow
 pytesseract
 SPARQLWrapper
diff --git a/setup.py b/setup.py
index 0010c05..3bdc180 100644
--- a/setup.py
+++ b/setup.py
@@ -38,6 +38,5 @@ setup(
         'pillow',
         'pytesseract',
         'shodan',
-        'misp_stix_converter'
     ],
 )

From cb29506640d5c4c9f20dcd0b7e1cd288cccbe9c3 Mon Sep 17 00:00:00 2001
From: Koen Van Impe <koen.vanimpe@cudeso.be>
Date: Sun, 27 Nov 2016 22:42:43 +0100
Subject: [PATCH 20/54] Extra VTI detections

---
 .../modules/import_mod/vmray_import.py        | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/misp_modules/modules/import_mod/vmray_import.py b/misp_modules/modules/import_mod/vmray_import.py
index a7b7110..a263294 100644
--- a/misp_modules/modules/import_mod/vmray_import.py
+++ b/misp_modules/modules/import_mod/vmray_import.py
@@ -174,6 +174,8 @@ def vmrayVtiPatterns(vti_patterns):
                 content = vmrayGeneric(pattern, "url", 1)
             elif pattern["category"] == "_network" and pattern["operation"] == "_connect":
                 content = vmrayConnect(pattern)
+            elif pattern["category"] == "_network" and pattern["operation"] == "_install_server":
+                content = vmrayGeneric(pattern)
 
             elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page":
                 content = vmrayGeneric(pattern)
@@ -181,6 +183,10 @@ def vmrayVtiPatterns(vti_patterns):
                 content = vmrayGeneric(pattern, "mutex", 1)
             elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_crashed_process":
                 content = vmrayGeneric(pattern)
+            elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_read_from_remote_process":
+                content = vmrayGeneric(pattern)
+            elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_create_process_with_hidden_window":
+                content = vmrayGeneric(pattern)
 
             elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
                 content = vmrayGeneric(pattern)
@@ -194,11 +200,20 @@ def vmrayVtiPatterns(vti_patterns):
 
             elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory":
                 content = vmrayGeneric(pattern)
+            elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory_system":
+                content = vmrayGeneric(pattern)
+            elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory_non_system":
+                content = vmrayGeneric(pattern)
             elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow":
                 content = vmrayGeneric(pattern)
+            elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow_non_system":
+                content = vmrayGeneric(pattern)
             elif only_network_info is False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files":
                 content = vmrayGeneric(pattern)
 
+            elif only_network_info is False and pattern["category"] == "_hide_tracks" and pattern["operation"] == "_hide_data_in_registry":
+                content = vmrayGeneric(pattern, "regkey", 1)
+
             elif only_network_info is False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script":
                 content = vmrayGeneric(pattern, "regkey", 1)
             elif only_network_info is False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges":
@@ -248,12 +263,14 @@ def vmrayGeneric(el, attr="", attrpos=1):
         content = el["technique_desc"]
         if content:
             if attr:
+                # Some elements are put between \"\" ; replace them to single
+                content = content.replace("\"\"","\"")
                 content_split = content.split("\"")
                 # Attributes are between open " and close "; so use >
                 if len(content_split) > attrpos:
                     content_split[attrpos] = vmraySanitizeInput(content_split[attrpos])
                     r["values"].append(content_split[attrpos])
-                    r["types"] = [attr]                
+                    r["types"] = [attr]
 
             # Adding the value also as text to get the extra description,
             # but this is pretty useless for "url"

From 48d38c28212c0b3ac8bb8ee324221d94b07e84ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Mon, 28 Nov 2016 18:12:31 +0100
Subject: [PATCH 21/54] Add initial Domain Tools module

---
 misp_modules/modules/expansion/domaintools.py | 124 ++++++++++++++++++
 1 file changed, 124 insertions(+)
 create mode 100755 misp_modules/modules/expansion/domaintools.py

diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py
new file mode 100755
index 0000000..19903df
--- /dev/null
+++ b/misp_modules/modules/expansion/domaintools.py
@@ -0,0 +1,124 @@
+import json
+import logging
+import sys
+
+from domaintools import API
+
+
+log = logging.getLogger('domaintools')
+log.setLevel(logging.DEBUG)
+ch = logging.StreamHandler(sys.stdout)
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ch.setFormatter(formatter)
+log.addHandler(ch)
+
+misperrors = {'error': 'Error'}
+mispattributes = {
+    'input': ['domain'],
+    'output': ['whois-registrant-email', 'whois-registrant-phone', 'whois-registrant-name',
+               'whois-registrar', 'whois-creation-date', 'freetext']
+}
+
+moduleinfo = {
+    'version': '0.1',
+    'author': 'Raphaël Vinot',
+    'description': 'DomainTools MISP expansion module.',
+    'module-type': ['expansion', 'hover']
+}
+
+moduleconfig = ['username', 'api_key']
+
+
+class DomainTools(object):
+
+    def __init__(self):
+        self.reg_mail = set()
+        self.reg_phone = set()
+        self.reg_name = set()
+        self.registrar = set()
+        self.creation_date = set()
+        self.freetext = ''
+
+    def dump(self):
+        to_return = []
+        if self.reg_mail:
+            to_return.append({'type': ['whois-registrant-email'], 'values': list(self.reg_mail)})
+        if self.reg_phone:
+            to_return.append({'type': ['whois-registrant-phone'], 'values': list(self.reg_phone)})
+        if self.reg_name:
+            to_return.append({'type': ['whois-registrant-name'], 'values': list(self.reg_name)})
+        if self.registrar:
+            to_return.append({'type': ['whois-registrar'], 'values': list(self.registrar)})
+        if self.creation_date:
+            to_return.append({'type': ['whois-creation-date'], 'values': list(self.creation_date)})
+        if self.freetext:
+            to_return.append({'type': ['freetext'], 'values': [self.freetext]})
+        return to_return
+
+
+def handler(q=False):
+    if not q:
+        return q
+
+    request = json.loads(q)
+    to_query = None
+    for t in mispattributes['input']:
+        to_query = request.get(t)
+        if to_query:
+            break
+    if not to_query:
+        misperrors['error'] = "Unsupported attributes type"
+        return misperrors
+
+    if request.get('config'):
+        if (request['config'].get('username') is None) or (request['config'].get('api_key') is None):
+            misperrors['error'] = 'DomainTools authentication is incomplete'
+            return misperrors
+        else:
+            domtools = API(request['config'].get('username'), request['config'].get('api_key'))
+    else:
+        misperrors['error'] = 'DomainTools authentication is missing'
+        return misperrors
+
+    whois_entry = domtools.parsed_whois(to_query)
+    values = DomainTools()
+
+    if whois_entry.has_key('error'):
+        misperrors['error'] = whois_entry['error']['message']
+        return misperrors
+
+    if whois_entry.has_key('registrant'):
+        values.reg_name.add(whois_entry['registrant'])
+
+    if whois_entry.has_key('registration'):
+        values.creation_date.add(whois_entry['registration']['created'])
+
+    if whois_entry.has_key('whois'):
+        values.freetext = whois_entry['whois']['record']
+    if whois_entry.emails():
+        # NOTE: not sure we want to do that (contains registrar emails)
+        values.reg_mail |= whois_entry.emails()
+    if whois_entry.has_key('parsed_whois'):
+        if whois_entry['parsed_whois']['created_date']:
+            values.creation_date.add(whois_entry['parsed_whois']['created_date'])
+        if whois_entry['parsed_whois']['registrar']['name']:
+            values.registrar.add(whois_entry['parsed_whois']['registrar']['name'])
+        for key, entry in whois_entry['parsed_whois']['contacts'].items():
+            # TODO: pass key as comment
+            if entry['email']:
+                values.reg_mail.add(entry['email'])
+            if entry['phone']:
+                values.reg_phone.add(entry['phone'])
+            if entry['name']:
+                values.reg_name.add(entry['name'])
+    return json.dumps({'results': values.dump()})
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

From 1e303b7af503e19d4e756d0d5d3441f8092e42a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <Rafiot@users.noreply.github.com>
Date: Tue, 29 Nov 2016 13:49:00 +0100
Subject: [PATCH 22/54] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b523140..87dba7d 100644
--- a/README.md
+++ b/README.md
@@ -46,8 +46,8 @@ sudo apt-get install python3-dev python3-pip libpq5
 cd /usr/local/src/
 sudo git clone https://github.com/MISP/misp-modules.git
 cd misp-modules
-sudo pip3 install --upgrade -r REQUIREMENTS
-sudo pip3 install --upgrade .
+sudo pip3 install -I -r REQUIREMENTS
+sudo pip3 install -I .
 sudo vi /etc/rc.local, add this line: `sudo -u www-data misp-modules -s &`
 ~~~~
 

From 7c6153478eef9f9342bb4fdcee9f13aa08bad62d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Wed, 30 Nov 2016 18:09:11 +0100
Subject: [PATCH 23/54] Add comments to fields when possible

---
 misp_modules/modules/expansion/domaintools.py | 60 ++++++++++++-------
 1 file changed, 40 insertions(+), 20 deletions(-)

diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py
index 19903df..1aeb7b6 100755
--- a/misp_modules/modules/expansion/domaintools.py
+++ b/misp_modules/modules/expansion/domaintools.py
@@ -33,27 +33,47 @@ moduleconfig = ['username', 'api_key']
 class DomainTools(object):
 
     def __init__(self):
-        self.reg_mail = set()
-        self.reg_phone = set()
-        self.reg_name = set()
+        self.reg_mail = {}
+        self.reg_phone = {}
+        self.reg_name = {}
         self.registrar = set()
         self.creation_date = set()
         self.freetext = ''
 
+    def _add_value(self, value_type, value, comment):
+        if value_type.get(value):
+            if comment:
+                value_type[value] += ' - {}'.format(comment)
+        else:
+            value_type[value] = comment
+        return value_type
+
+    def add_mail(self, mail, comment=None):
+        self.reg_mail = self._add_value(self.reg_mail, mail, comment)
+
+    def add_phone(self, phone, comment=None):
+        self.reg_phone = self._add_value(self.reg_phone, phone, comment)
+
+    def add_name(self, name, comment=None):
+        self.reg_name = self._add_value(self.reg_name, name, comment)
+
     def dump(self):
         to_return = []
         if self.reg_mail:
-            to_return.append({'type': ['whois-registrant-email'], 'values': list(self.reg_mail)})
+            for mail, comment in self.reg_mail.items():
+                to_return.append({'type': ['whois-registrant-email'], 'values': [mail], 'comment': comment})
         if self.reg_phone:
-            to_return.append({'type': ['whois-registrant-phone'], 'values': list(self.reg_phone)})
+            for phone, comment in self.reg_phone.items():
+                to_return.append({'type': ['whois-registrant-phone'], 'values': [phone], 'comment': comment})
         if self.reg_name:
-            to_return.append({'type': ['whois-registrant-name'], 'values': list(self.reg_name)})
+            for name, comment in self.reg_name.items():
+                to_return.append({'type': ['whois-registrant-name'], 'values': [name], 'comment': comment})
         if self.registrar:
             to_return.append({'type': ['whois-registrar'], 'values': list(self.registrar)})
         if self.creation_date:
             to_return.append({'type': ['whois-creation-date'], 'values': list(self.creation_date)})
         if self.freetext:
-            to_return.append({'type': ['freetext'], 'values': [self.freetext]})
+            to_return.append({'type': ['freetext'], 'values': [self.freetext], 'comment': 'Freetext import'})
         return to_return
 
 
@@ -82,36 +102,36 @@ def handler(q=False):
         return misperrors
 
     whois_entry = domtools.parsed_whois(to_query)
+    print(whois_entry)
     values = DomainTools()
 
-    if whois_entry.has_key('error'):
+    if whois_entry.get('error'):
         misperrors['error'] = whois_entry['error']['message']
         return misperrors
 
-    if whois_entry.has_key('registrant'):
-        values.reg_name.add(whois_entry['registrant'])
+    if whois_entry.get('registrant'):
+        values.add_name(whois_entry['registrant'], 'Parsed registrant')
 
-    if whois_entry.has_key('registration'):
+    if whois_entry.get('registration'):
         values.creation_date.add(whois_entry['registration']['created'])
 
-    if whois_entry.has_key('whois'):
+    if whois_entry.get('whois'):
         values.freetext = whois_entry['whois']['record']
-    if whois_entry.emails():
-        # NOTE: not sure we want to do that (contains registrar emails)
-        values.reg_mail |= whois_entry.emails()
-    if whois_entry.has_key('parsed_whois'):
+    if whois_entry.get('parsed_whois'):
         if whois_entry['parsed_whois']['created_date']:
             values.creation_date.add(whois_entry['parsed_whois']['created_date'])
         if whois_entry['parsed_whois']['registrar']['name']:
             values.registrar.add(whois_entry['parsed_whois']['registrar']['name'])
         for key, entry in whois_entry['parsed_whois']['contacts'].items():
-            # TODO: pass key as comment
             if entry['email']:
-                values.reg_mail.add(entry['email'])
+                values.add_mail(entry['email'], key)
             if entry['phone']:
-                values.reg_phone.add(entry['phone'])
+                values.add_phone(entry['phone'], key)
             if entry['name']:
-                values.reg_name.add(entry['name'])
+                values.add_name(entry['name'], key)
+    if whois_entry.emails():
+        for mail in whois_entry.emails():
+            values.add_mail(mail)
     return json.dumps({'results': values.dump()})
 
 

From afd8b71349270ad9a7054931cdb51fa2ff040b47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 1 Dec 2016 10:26:40 +0100
Subject: [PATCH 24/54] Avoid passing None in comments

---
 misp_modules/modules/expansion/domaintools.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py
index 1aeb7b6..521100e 100755
--- a/misp_modules/modules/expansion/domaintools.py
+++ b/misp_modules/modules/expansion/domaintools.py
@@ -61,13 +61,13 @@ class DomainTools(object):
         to_return = []
         if self.reg_mail:
             for mail, comment in self.reg_mail.items():
-                to_return.append({'type': ['whois-registrant-email'], 'values': [mail], 'comment': comment})
+                to_return.append({'type': ['whois-registrant-email'], 'values': [mail], 'comment': comment or ''})
         if self.reg_phone:
             for phone, comment in self.reg_phone.items():
-                to_return.append({'type': ['whois-registrant-phone'], 'values': [phone], 'comment': comment})
+                to_return.append({'type': ['whois-registrant-phone'], 'values': [phone], 'comment': comment or ''})
         if self.reg_name:
             for name, comment in self.reg_name.items():
-                to_return.append({'type': ['whois-registrant-name'], 'values': [name], 'comment': comment})
+                to_return.append({'type': ['whois-registrant-name'], 'values': [name], 'comment': comment or ''})
         if self.registrar:
             to_return.append({'type': ['whois-registrar'], 'values': list(self.registrar)})
         if self.creation_date:

From 6db5436c6287802af053bea4c490207c60c9265e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 1 Dec 2016 11:54:04 +0100
Subject: [PATCH 25/54] remove json.dumps

---
 misp_modules/modules/expansion/domaintools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py
index 521100e..f483b38 100755
--- a/misp_modules/modules/expansion/domaintools.py
+++ b/misp_modules/modules/expansion/domaintools.py
@@ -132,7 +132,7 @@ def handler(q=False):
     if whois_entry.emails():
         for mail in whois_entry.emails():
             values.add_mail(mail)
-    return json.dumps({'results': values.dump()})
+    return {'results': values.dump()}
 
 
 def introspection():

From 9dbd241e636104b41ccf243582395c1040cfa607 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 1 Dec 2016 12:14:16 +0100
Subject: [PATCH 26/54] fix typo

---
 misp_modules/modules/expansion/domaintools.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py
index f483b38..2e8262e 100755
--- a/misp_modules/modules/expansion/domaintools.py
+++ b/misp_modules/modules/expansion/domaintools.py
@@ -61,19 +61,19 @@ class DomainTools(object):
         to_return = []
         if self.reg_mail:
             for mail, comment in self.reg_mail.items():
-                to_return.append({'type': ['whois-registrant-email'], 'values': [mail], 'comment': comment or ''})
+                to_return.append({'types': ['whois-registrant-email'], 'values': [mail], 'comment': comment or ''})
         if self.reg_phone:
             for phone, comment in self.reg_phone.items():
-                to_return.append({'type': ['whois-registrant-phone'], 'values': [phone], 'comment': comment or ''})
+                to_return.append({'types': ['whois-registrant-phone'], 'values': [phone], 'comment': comment or ''})
         if self.reg_name:
             for name, comment in self.reg_name.items():
-                to_return.append({'type': ['whois-registrant-name'], 'values': [name], 'comment': comment or ''})
+                to_return.append({'types': ['whois-registrant-name'], 'values': [name], 'comment': comment or ''})
         if self.registrar:
-            to_return.append({'type': ['whois-registrar'], 'values': list(self.registrar)})
+            to_return.append({'types': ['whois-registrar'], 'values': list(self.registrar)})
         if self.creation_date:
-            to_return.append({'type': ['whois-creation-date'], 'values': list(self.creation_date)})
+            to_return.append({'types': ['whois-creation-date'], 'values': list(self.creation_date)})
         if self.freetext:
-            to_return.append({'type': ['freetext'], 'values': [self.freetext], 'comment': 'Freetext import'})
+            to_return.append({'types': ['freetext'], 'values': [self.freetext], 'comment': 'Freetext import'})
         return to_return
 
 

From 04b9da883cc2208a98d4ba825917ecdbf7ec79e2 Mon Sep 17 00:00:00 2001
From: Alexander J <deralexxx@users.noreply.github.com>
Date: Thu, 1 Dec 2016 13:10:23 +0100
Subject: [PATCH 27/54] Update README.md

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 87dba7d..30637f6 100644
--- a/README.md
+++ b/README.md
@@ -49,6 +49,7 @@ cd misp-modules
 sudo pip3 install -I -r REQUIREMENTS
 sudo pip3 install -I .
 sudo vi /etc/rc.local, add this line: `sudo -u www-data misp-modules -s &`
+/usr/local/bin/misp-modules #to start the modules
 ~~~~
 
 ## How to add your own MISP modules?

From 7db1216efb6685ec443d2e9d661dbfe8c30e7a7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 1 Dec 2016 13:43:37 +0100
Subject: [PATCH 28/54] Add more comments

---
 misp_modules/modules/expansion/domaintools.py | 39 ++++++++++++-------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py
index 2e8262e..94a2e10 100755
--- a/misp_modules/modules/expansion/domaintools.py
+++ b/misp_modules/modules/expansion/domaintools.py
@@ -36,8 +36,8 @@ class DomainTools(object):
         self.reg_mail = {}
         self.reg_phone = {}
         self.reg_name = {}
-        self.registrar = set()
-        self.creation_date = set()
+        self.registrar = {}
+        self.creation_date = {}
         self.freetext = ''
 
     def _add_value(self, value_type, value, comment):
@@ -45,7 +45,7 @@ class DomainTools(object):
             if comment:
                 value_type[value] += ' - {}'.format(comment)
         else:
-            value_type[value] = comment
+            value_type[value] = comment or ''
         return value_type
 
     def add_mail(self, mail, comment=None):
@@ -57,23 +57,31 @@ class DomainTools(object):
     def add_name(self, name, comment=None):
         self.reg_name = self._add_value(self.reg_name, name, comment)
 
+    def add_registrar(self, reg, comment=None):
+        self.registrar = self._add_value(self.registrar, reg, comment)
+
+    def add_creation_date(self, date, comment=None):
+        self.creation_date = self._add_value(self.creation_date, date, comment)
+
     def dump(self):
         to_return = []
         if self.reg_mail:
             for mail, comment in self.reg_mail.items():
-                to_return.append({'types': ['whois-registrant-email'], 'values': [mail], 'comment': comment or ''})
+                to_return.append({'type': 'whois-registrant-email', 'values': [mail], 'comment': comment or ''})
         if self.reg_phone:
             for phone, comment in self.reg_phone.items():
-                to_return.append({'types': ['whois-registrant-phone'], 'values': [phone], 'comment': comment or ''})
+                to_return.append({'type': 'whois-registrant-phone', 'values': [phone], 'comment': comment or ''})
         if self.reg_name:
             for name, comment in self.reg_name.items():
-                to_return.append({'types': ['whois-registrant-name'], 'values': [name], 'comment': comment or ''})
+                to_return.append({'type': 'whois-registrant-name', 'values': [name], 'comment': comment or ''})
         if self.registrar:
-            to_return.append({'types': ['whois-registrar'], 'values': list(self.registrar)})
+            for reg, comment in self.registrar.items():
+                to_return.append({'type': 'whois-registrar', 'values': [reg], 'comment': comment or ''})
         if self.creation_date:
-            to_return.append({'types': ['whois-creation-date'], 'values': list(self.creation_date)})
+            for date, comment in self.creation_date.items():
+                to_return.append({'type': 'whois-creation-date', 'values': [date], 'comment': comment or ''})
         if self.freetext:
-            to_return.append({'types': ['freetext'], 'values': [self.freetext], 'comment': 'Freetext import'})
+            to_return.append({'type': 'freetext', 'values': [self.freetext], 'comment': 'Freetext import'})
         return to_return
 
 
@@ -113,15 +121,19 @@ def handler(q=False):
         values.add_name(whois_entry['registrant'], 'Parsed registrant')
 
     if whois_entry.get('registration'):
-        values.creation_date.add(whois_entry['registration']['created'])
+        values.add_creation_date(whois_entry['registration']['created'], 'timestamp')
 
     if whois_entry.get('whois'):
         values.freetext = whois_entry['whois']['record']
     if whois_entry.get('parsed_whois'):
         if whois_entry['parsed_whois']['created_date']:
-            values.creation_date.add(whois_entry['parsed_whois']['created_date'])
+            values.add_creation_date(whois_entry['parsed_whois']['created_date'])
         if whois_entry['parsed_whois']['registrar']['name']:
-            values.registrar.add(whois_entry['parsed_whois']['registrar']['name'])
+            values.add_registrar(whois_entry['parsed_whois']['registrar']['name'], 'name')
+        if whois_entry['parsed_whois']['registrar']['url']:
+            values.add_registrar(whois_entry['parsed_whois']['registrar']['url'], 'url')
+        if whois_entry['parsed_whois']['registrar']['iana_id']:
+            values.add_registrar(whois_entry['parsed_whois']['registrar']['iana_id'], 'iana_id')
         for key, entry in whois_entry['parsed_whois']['contacts'].items():
             if entry['email']:
                 values.add_mail(entry['email'], key)
@@ -131,7 +143,8 @@ def handler(q=False):
                 values.add_name(entry['name'], key)
     if whois_entry.emails():
         for mail in whois_entry.emails():
-            values.add_mail(mail)
+            if mail not in values.reg_mail.keys():
+                values.add_mail(mail, 'Maybe registrar')
     return {'results': values.dump()}
 
 

From 163730469174858d8563bf3d9ae7106efe8eb52b Mon Sep 17 00:00:00 2001
From: Alexander J <deralexxx@users.noreply.github.com>
Date: Thu, 1 Dec 2016 14:59:42 +0100
Subject: [PATCH 29/54] Update README.md

mentioning import / export modules
---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 87dba7d..48b44f1 100644
--- a/README.md
+++ b/README.md
@@ -53,7 +53,7 @@ sudo vi /etc/rc.local, add this line: `sudo -u www-data misp-modules -s &`
 
 ## How to add your own MISP modules?
 
-Create your module in [misp_modules/modules/expansion/](misp_modules/modules/expansion/). The module should have at minimum three functions:
+Create your module in [misp_modules/modules/](misp_modules/modules/). The module should have at minimum three functions:
 
 * **introspection** function that returns a dict of the supported attributes (input and output) by your expansion module.
 * **handler** function which accepts a JSON document to expand the values and return a dictionary of the expanded values.
@@ -80,10 +80,11 @@ If your module requires additional configuration (to be exposed via the MISP use
 
 ### Module type
 
-A MISP module can be of two types:
+A MISP module can be of different types:
 
 - **expansion** - service related to an attribute that can be used to extend and update an existing event.
 - **hover** - service related to an attribute to provide additional information to the users without updating the event.
+- **import/export** - import / export data
 
 module-type is an array where the list of supported types can be added.
 

From 17205a19134626d912ca1d5e0800912d3a9a078b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 1 Dec 2016 16:41:50 +0100
Subject: [PATCH 30/54] Add domain profile and reputation

---
 misp_modules/modules/expansion/domaintools.py | 41 +++++++++++++++++--
 1 file changed, 38 insertions(+), 3 deletions(-)

diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py
index 94a2e10..79b345e 100755
--- a/misp_modules/modules/expansion/domaintools.py
+++ b/misp_modules/modules/expansion/domaintools.py
@@ -38,11 +38,13 @@ class DomainTools(object):
         self.reg_name = {}
         self.registrar = {}
         self.creation_date = {}
+        self.domain_ip = {}
+        self.risk = ()
         self.freetext = ''
 
     def _add_value(self, value_type, value, comment):
         if value_type.get(value):
-            if comment:
+            if comment and comment not in value_type[value]:
                 value_type[value] += ' - {}'.format(comment)
         else:
             value_type[value] = comment or ''
@@ -63,6 +65,9 @@ class DomainTools(object):
     def add_creation_date(self, date, comment=None):
         self.creation_date = self._add_value(self.creation_date, date, comment)
 
+    def add_ip(self, ip, comment=None):
+        self.domain_ip = self._add_value(self.domain_ip, ip, comment)
+
     def dump(self):
         to_return = []
         if self.reg_mail:
@@ -80,8 +85,13 @@ class DomainTools(object):
         if self.creation_date:
             for date, comment in self.creation_date.items():
                 to_return.append({'type': 'whois-creation-date', 'values': [date], 'comment': comment or ''})
+        if self.domain_ip:
+            for ip, comment in self.domain_ip.items():
+                to_return.append({'types': ['dst-ip', 'src-ip'], 'values': [ip], 'comment': comment or ''})
         if self.freetext:
             to_return.append({'type': 'freetext', 'values': [self.freetext], 'comment': 'Freetext import'})
+        if self.risk:
+            to_return.append({'type': 'text', 'values': [self.risk[0]], 'comment': self.risk[1]})
         return to_return
 
 
@@ -110,15 +120,40 @@ def handler(q=False):
         return misperrors
 
     whois_entry = domtools.parsed_whois(to_query)
-    print(whois_entry)
+    profile = domtools.domain_profile(to_query)
+    # NOTE: profile['website_data']['response_code'] could be used to see if the host is still up. Maybe set a tag.
+    reputation = domtools.reputation(to_query, include_reasons=True)
+    # NOTE: use that value in a tag when we will have attribute level tagging
     values = DomainTools()
 
     if whois_entry.get('error'):
         misperrors['error'] = whois_entry['error']['message']
         return misperrors
 
+    if profile.get('error'):
+        misperrors['error'] = profile['error']['message']
+        return misperrors
+
+    if reputation and not reputation.get('error'):
+        reasons = ', '.join(reputation['reasons'])
+        values.risk = [reputation['risk_score'], 'Risk value of {} (via Domain Tools), Reasons: {}'.format(to_query, reasons)]
+
     if whois_entry.get('registrant'):
         values.add_name(whois_entry['registrant'], 'Parsed registrant')
+    if profile.get('registrant'):
+        values.add_name(profile['registrant']['name'], 'Profile registrant')
+
+    if profile.get('server'):
+        other_domains = profile['server']['other_domains']
+        values.add_ip(profile['server']['ip_address'], 'IP of {} (via DomainTools). Has {} other domains.'.format(to_query, other_domains))
+
+    if profile.get('registration'):
+        if profile['registration'].get('created'):
+            values.add_creation_date(profile['registration']['created'], 'created')
+        if profile['registration'].get('updated'):
+            values.add_creation_date(profile['registration']['updated'], 'updated')
+        if profile['registration'].get('registrar'):
+            values.add_registrar(profile['registration']['registrar'], 'name')
 
     if whois_entry.get('registration'):
         values.add_creation_date(whois_entry['registration']['created'], 'timestamp')
@@ -127,7 +162,7 @@ def handler(q=False):
         values.freetext = whois_entry['whois']['record']
     if whois_entry.get('parsed_whois'):
         if whois_entry['parsed_whois']['created_date']:
-            values.add_creation_date(whois_entry['parsed_whois']['created_date'])
+            values.add_creation_date(whois_entry['parsed_whois']['created_date'], 'created')
         if whois_entry['parsed_whois']['registrar']['name']:
             values.add_registrar(whois_entry['parsed_whois']['registrar']['name'], 'name')
         if whois_entry['parsed_whois']['registrar']['url']:

From 0f8fa4aaec6abc899eab1403a06e8194d403788d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 1 Dec 2016 16:44:29 +0100
Subject: [PATCH 31/54] Fix Typo

---
 misp_modules/modules/expansion/domaintools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py
index 79b345e..6382726 100755
--- a/misp_modules/modules/expansion/domaintools.py
+++ b/misp_modules/modules/expansion/domaintools.py
@@ -87,7 +87,7 @@ class DomainTools(object):
                 to_return.append({'type': 'whois-creation-date', 'values': [date], 'comment': comment or ''})
         if self.domain_ip:
             for ip, comment in self.domain_ip.items():
-                to_return.append({'types': ['dst-ip', 'src-ip'], 'values': [ip], 'comment': comment or ''})
+                to_return.append({'types': ['ip-dst', 'ip-src'], 'values': [ip], 'comment': comment or ''})
         if self.freetext:
             to_return.append({'type': 'freetext', 'values': [self.freetext], 'comment': 'Freetext import'})
         if self.risk:

From 2e3119b5f45a65f585e34b1239764d73b41c65fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 1 Dec 2016 17:36:40 +0100
Subject: [PATCH 32/54] Add domaintools to the import list

---
 misp_modules/modules/expansion/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py
index 62a6ecd..548eb82 100644
--- a/misp_modules/modules/expansion/__init__.py
+++ b/misp_modules/modules/expansion/__init__.py
@@ -1,4 +1,5 @@
 from . import _vmray
 
-__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
-           'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'wiki']
+__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl',
+           'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'ipasn', 'passivetotal', 'sourcecache',
+           'virustotal', 'whois', 'shodan', 'reversedns', 'wiki']

From a14c5b749a62c73774fec9070c131f44361684bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 1 Dec 2016 17:42:10 +0100
Subject: [PATCH 33/54] Update requirements list

---
 REQUIREMENTS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/REQUIREMENTS b/REQUIREMENTS
index 16f5512..8340c82 100644
--- a/REQUIREMENTS
+++ b/REQUIREMENTS
@@ -17,3 +17,4 @@ git+https://github.com/CIRCL/PyMISP.git#egg=pymisp
 pillow
 pytesseract
 SPARQLWrapper
+domaintools_api

From 2b020c55baf3a7ed074ff60010e2b2fc97a84a2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Fri, 2 Dec 2016 15:29:44 +0100
Subject: [PATCH 34/54] Add test for domaintools

---
 tests/test.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/test.py b/tests/test.py
index faf5173..065e353 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -7,7 +7,6 @@ import base64
 import json
 import os
 
-
 class TestModules(unittest.TestCase):
 
     def setUp(self):
@@ -59,5 +58,11 @@ class TestModules(unittest.TestCase):
             response = requests.post(self.url + "query", data=f.read()).json()
         assert(response)
 
+    def test_domaintools(self):
+        query = {'config': {'username': 'test_user', 'api_key': 'test_key'}, 'module': 'domaintools', 'domain': 'domaintools.com'}
+        response = requests.post(self.url + "query", data=json.dumps(query)).json()
+        print(response)
+
+
 if __name__ == '__main__':
     unittest.main()

From f8bedd4554515936645c1e1eeb88963a25324f52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Fri, 2 Dec 2016 16:16:25 +0100
Subject: [PATCH 35/54] Remove domaintools tests

---
 tests/test.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 065e353..aaa410a 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -58,10 +58,14 @@ class TestModules(unittest.TestCase):
             response = requests.post(self.url + "query", data=f.read()).json()
         assert(response)
 
-    def test_domaintools(self):
-        query = {'config': {'username': 'test_user', 'api_key': 'test_key'}, 'module': 'domaintools', 'domain': 'domaintools.com'}
-        response = requests.post(self.url + "query", data=json.dumps(query)).json()
-        print(response)
+    #def test_domaintools(self):
+    #    query = {'config': {'username': 'test_user', 'api_key': 'test_key'}, 'module': 'domaintools', 'domain': 'domaintools.com'}
+    #    try:
+    #        response = requests.post(self.url + "query", data=json.dumps(query)).json()
+    #    except:
+    #        pass
+    #    response = requests.post(self.url + "query", data=json.dumps(query)).json()
+    #    print(response)
 
 
 if __name__ == '__main__':

From 1bb2b1080061323152c9f6e4233682a63f359829 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Fri, 2 Dec 2016 17:12:21 +0100
Subject: [PATCH 36/54] DomainTools module added

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index c99c555..e7da93d 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
 * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
 * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
+* [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
 * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
 * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
 * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.

From f8c7271467d84159743d02b1b699fb41bea7e079 Mon Sep 17 00:00:00 2001
From: Andreas Muehlemann <andreas.muehlemann@switch.ch>
Date: Wed, 7 Dec 2016 14:18:21 +0100
Subject: [PATCH 37/54] added config option to geoip_country.py

---
 misp_modules/modules/expansion/geoip_country.cfg |  2 ++
 misp_modules/modules/expansion/geoip_country.py  | 13 +++++--------
 2 files changed, 7 insertions(+), 8 deletions(-)
 create mode 100644 misp_modules/modules/expansion/geoip_country.cfg

diff --git a/misp_modules/modules/expansion/geoip_country.cfg b/misp_modules/modules/expansion/geoip_country.cfg
new file mode 100644
index 0000000..ab12f90
--- /dev/null
+++ b/misp_modules/modules/expansion/geoip_country.cfg
@@ -0,0 +1,2 @@
+[defaults]
+database=/opt/misp-modules/var/GeoIP.dat
diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py
index 01e95de..069f20a 100644
--- a/misp_modules/modules/expansion/geoip_country.py
+++ b/misp_modules/modules/expansion/geoip_country.py
@@ -1,5 +1,6 @@
 import json, pygeoip
 import sys, logging
+import ConfigParser
 
 log = logging.getLogger('geoip_country')
 log.setLevel(logging.DEBUG)
@@ -21,7 +22,9 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
 moduleconfig = ['database']
 
 # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
-gi = pygeoip.GeoIP('/opt/misp-modules/var/GeoIP.dat')
+config = ConfigParser.ConfigParser()
+config.read('./geoip_contry.cfg')
+gi = pygeoip.GeoIP(config.get('defaults', 'database')
 
 def handler(q=False):
     if q is False:
@@ -41,12 +44,6 @@ def handler(q=False):
 
     log.debug(toquery)
 
-    #if request.get('config'):
-    #    if request['config'].get('database'):
-    #        gi = pygeoip.GeoIP(request['config'].get('database'))
-    #else:
-    #    gi = pygeoip.GeoIP('/opt/misp-modules/var/GeoIP.dat')
-
     try:
         answer = gi.country_code_by_addr(toquery)
     except:
@@ -55,7 +52,7 @@ def handler(q=False):
 
     r = {'results': [{'types': mispattributes['output'],
                       'values': [str(answer)]}]}
-                     
+
     return r
 
 def introspection():

From 6ea7acc5e4983841e87b00345c03be4585a5b632 Mon Sep 17 00:00:00 2001
From: Andreas Muehlemann <andreas.muehlemann@switch.ch>
Date: Wed, 7 Dec 2016 14:28:27 +0100
Subject: [PATCH 38/54] removed debug message

---
 misp_modules/modules/expansion/geoip_country.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py
index 069f20a..85dd2d8 100644
--- a/misp_modules/modules/expansion/geoip_country.py
+++ b/misp_modules/modules/expansion/geoip_country.py
@@ -31,8 +31,6 @@ def handler(q=False):
         return False
     request = json.loads(q)
 
-    log.debug("BLA")
-
     if request.get('ip-dst'):
         toquery = request['ip-dst']
     elif request.get('ip-src'):
@@ -61,3 +59,4 @@ def introspection():
 def version():
     moduleinfo['config'] = moduleconfig
     return moduleinfo
+

From d09c2f3d44d12025b0e8314e75c3cee63267fc0e Mon Sep 17 00:00:00 2001
From: Andreas Muehlemann <andreas.muehlemann@switch.ch>
Date: Wed, 7 Dec 2016 14:29:11 +0100
Subject: [PATCH 39/54] removed unneeded config option for misp

---
 misp_modules/modules/expansion/geoip_country.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py
index 85dd2d8..ae9ccc9 100644
--- a/misp_modules/modules/expansion/geoip_country.py
+++ b/misp_modules/modules/expansion/geoip_country.py
@@ -18,9 +18,6 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
               'description': 'Query a local copy of Maxminds Geolite database',
               'module-type': ['expansion', 'hover']}
 
-# config fields that your code expects from the site admin
-moduleconfig = ['database']
-
 # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
 config = ConfigParser.ConfigParser()
 config.read('./geoip_contry.cfg')
@@ -59,4 +56,3 @@ def introspection():
 def version():
     moduleinfo['config'] = moduleconfig
     return moduleinfo
-

From 1e1796b41411c8e17088095fdb457533b364831f Mon Sep 17 00:00:00 2001
From: Andreas Muehlemann <andreas.muehlemann@switch.ch>
Date: Wed, 7 Dec 2016 15:19:54 +0100
Subject: [PATCH 40/54] updated missing parenthesis

---
 misp_modules/modules/expansion/geoip_country.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py
index ae9ccc9..4ffc0b9 100644
--- a/misp_modules/modules/expansion/geoip_country.py
+++ b/misp_modules/modules/expansion/geoip_country.py
@@ -21,7 +21,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
 # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
 config = ConfigParser.ConfigParser()
 config.read('./geoip_contry.cfg')
-gi = pygeoip.GeoIP(config.get('defaults', 'database')
+gi = pygeoip.GeoIP(config.get('defaults', 'database'))
 
 def handler(q=False):
     if q is False:

From a95af2642455f3d99b055c43b5a143877d32ca05 Mon Sep 17 00:00:00 2001
From: Andreas Muehlemann <andreas.muehlemann@switch.ch>
Date: Wed, 7 Dec 2016 15:30:49 +0100
Subject: [PATCH 41/54] changed configparser from python2 to python3

---
 misp_modules/modules/expansion/geoip_country.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py
index 4ffc0b9..634d4c7 100644
--- a/misp_modules/modules/expansion/geoip_country.py
+++ b/misp_modules/modules/expansion/geoip_country.py
@@ -1,6 +1,6 @@
 import json, pygeoip
 import sys, logging
-import ConfigParser
+import configparser
 
 log = logging.getLogger('geoip_country')
 log.setLevel(logging.DEBUG)
@@ -19,7 +19,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
               'module-type': ['expansion', 'hover']}
 
 # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
-config = ConfigParser.ConfigParser()
+config = configparser.ConfigParser()
 config.read('./geoip_contry.cfg')
 gi = pygeoip.GeoIP(config.get('defaults', 'database'))
 

From 6dcc77ba5d247b1aed1bf22e3866b8bb94a9b37a Mon Sep 17 00:00:00 2001
From: Andreas Muehlemann <andreas.muehlemann@switch.ch>
Date: Wed, 7 Dec 2016 15:48:08 +0100
Subject: [PATCH 42/54] fixed typo

---
 misp_modules/modules/expansion/geoip_country.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py
index 634d4c7..a04d739 100644
--- a/misp_modules/modules/expansion/geoip_country.py
+++ b/misp_modules/modules/expansion/geoip_country.py
@@ -20,7 +20,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
 
 # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
 config = configparser.ConfigParser()
-config.read('./geoip_contry.cfg')
+config.read('geoip_country.cfg')
 gi = pygeoip.GeoIP(config.get('defaults', 'database'))
 
 def handler(q=False):

From 6853d67a43e4aa04ca51dc6e2e57b6e3b8654db5 Mon Sep 17 00:00:00 2001
From: Andreas Muehlemann <andreas.muehlemann@switch.ch>
Date: Wed, 7 Dec 2016 16:13:46 +0100
Subject: [PATCH 43/54] fixed more typos

---
 misp_modules/modules/expansion/geoip_country.cfg | 2 +-
 misp_modules/modules/expansion/geoip_country.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/misp_modules/modules/expansion/geoip_country.cfg b/misp_modules/modules/expansion/geoip_country.cfg
index ab12f90..bcd1c85 100644
--- a/misp_modules/modules/expansion/geoip_country.cfg
+++ b/misp_modules/modules/expansion/geoip_country.cfg
@@ -1,2 +1,2 @@
-[defaults]
+[DEFAULT]
 database=/opt/misp-modules/var/GeoIP.dat
diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py
index a04d739..cac709c 100644
--- a/misp_modules/modules/expansion/geoip_country.py
+++ b/misp_modules/modules/expansion/geoip_country.py
@@ -21,7 +21,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
 # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
 config = configparser.ConfigParser()
 config.read('geoip_country.cfg')
-gi = pygeoip.GeoIP(config.get('defaults', 'database'))
+gi = pygeoip.GeoIP(config.get('DEFAULT', 'database'))
 
 def handler(q=False):
     if q is False:

From 98a27ac3ff9743e17163d65f8fd86dc9ff5dfcdc Mon Sep 17 00:00:00 2001
From: Andreas Muehlemann <andreas.muehlemann@switch.ch>
Date: Wed, 7 Dec 2016 16:36:02 +0100
Subject: [PATCH 44/54] removed DEFAULT section from configfile

---
 misp_modules/modules/expansion/geoip_country.cfg | 4 ++--
 misp_modules/modules/expansion/geoip_country.py  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/misp_modules/modules/expansion/geoip_country.cfg b/misp_modules/modules/expansion/geoip_country.cfg
index bcd1c85..f27e7d9 100644
--- a/misp_modules/modules/expansion/geoip_country.cfg
+++ b/misp_modules/modules/expansion/geoip_country.cfg
@@ -1,2 +1,2 @@
-[DEFAULT]
-database=/opt/misp-modules/var/GeoIP.dat
+[GEOIP]
+database: /opt/misp-modules/var/GeoIP.dat
diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py
index cac709c..f5a1984 100644
--- a/misp_modules/modules/expansion/geoip_country.py
+++ b/misp_modules/modules/expansion/geoip_country.py
@@ -21,7 +21,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
 # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
 config = configparser.ConfigParser()
 config.read('geoip_country.cfg')
-gi = pygeoip.GeoIP(config.get('DEFAULT', 'database'))
+gi = pygeoip.GeoIP(config.get('GEOIP', 'database'))
 
 def handler(q=False):
     if q is False:

From cc58b05d6e1c24468a0770cfccb7a789053163d8 Mon Sep 17 00:00:00 2001
From: Andreas Muehlemann <andreas.muehlemann@switch.ch>
Date: Wed, 7 Dec 2016 17:28:16 +0100
Subject: [PATCH 45/54] added empty line to end of config file

---
 misp_modules/modules/expansion/geoip_country.cfg | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/misp_modules/modules/expansion/geoip_country.cfg b/misp_modules/modules/expansion/geoip_country.cfg
index f27e7d9..95037e5 100644
--- a/misp_modules/modules/expansion/geoip_country.cfg
+++ b/misp_modules/modules/expansion/geoip_country.cfg
@@ -1,2 +1,3 @@
 [GEOIP]
-database: /opt/misp-modules/var/GeoIP.dat
+database = /opt/misp-modules/var/GeoIP.dat
+

From b76f59edcbaeb0fe21043a615a54f0c59287f215 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@misp.leet.fyi>
Date: Wed, 7 Dec 2016 16:36:31 +0000
Subject: [PATCH 46/54] Added cuckooimport.py

---
 misp_modules/modules/import_mod/__init__.py   |   2 +-
 .../modules/import_mod/cuckooimport.py        | 196 ++++++++++++++++++
 2 files changed, 197 insertions(+), 1 deletion(-)
 create mode 100755 misp_modules/modules/import_mod/cuckooimport.py

diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py
index 70b674a..b614d3f 100644
--- a/misp_modules/modules/import_mod/__init__.py
+++ b/misp_modules/modules/import_mod/__init__.py
@@ -1,3 +1,3 @@
 from . import _vmray
 
-__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport']
+__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport']
diff --git a/misp_modules/modules/import_mod/cuckooimport.py b/misp_modules/modules/import_mod/cuckooimport.py
new file mode 100755
index 0000000..193477f
--- /dev/null
+++ b/misp_modules/modules/import_mod/cuckooimport.py
@@ -0,0 +1,196 @@
+import json
+import logging 
+import sys 
+import base64 
+
+misperrors = {'error': 'Error'}
+userConfig = {}
+inputSource = ['file']
+
+moduleinfo = {'version': '0.1', 'author': 'Victor van der Stoep',
+              'description': 'Cuckoo JSON import',
+              'module-type': ['import']}
+
+moduleconfig = []
+
+def handler(q=False):
+    # Just in case we have no data
+    if q is False:
+        return False
+    
+    # The return value
+    r = {'results': []}
+
+    # Load up that JSON
+    q = json.loads(q) 
+    data = base64.b64decode(q.get("data")).decode('utf-8')
+    
+    # If something really weird happened
+    if not data:
+        return json.dumps({"success": 0})
+   
+    data = json.loads(data)
+    
+    # Get characteristics of file  
+    targetFile = data['target']['file']
+    
+    # Process the inital binary
+    processBinary(r, targetFile, initial = True)
+    
+    # Get binary information for dropped files
+    if(data.get('dropped')):
+        for droppedFile in data['dropped']:
+            processBinary(r, droppedFile, dropped = True)
+    
+    # Add malscore to results
+    r["results"].append({ 
+        "values": "Malscore: {} ".format(data['malscore']), 
+        "types": "comment",
+        "categories": "Payload delivery", 
+        "comment": "Cuckoo analysis: MalScore"
+    })
+    
+    # Add virustotal data, if exists
+    if(data.get('virustotal')):
+        processVT(r, data['virustotal'])
+    
+    # Add network information, should be improved
+    processNetwork(r, data['network'])
+    
+    # Add behavioral information
+    processSummary(r, data['behavior']['summary'])
+    
+    # Return 
+    return r
+
+def processSummary(r, summary):
+    r["results"].append({ 
+        "values": summary['mutexes'], 
+        "types": "mutex",
+        "categories": "Artifacts dropped", 
+        "comment": "Cuckoo analysis: Observed mutexes"
+    })
+    
+def processVT(r, virustotal):
+    category = "Antivirus detection"
+    comment = "VirusTotal analysis" 
+    
+    if(virustotal.get('permalink')):
+        r["results"].append({ 
+            "values": virustotal['permalink'], 
+            "types": "link",
+            "categories": category, 
+            "comments": comment + " - Permalink"
+        })
+    
+    if(virustotal.get('total')):
+        r["results"].append({ 
+            "values": "VirusTotal detection rate {}/{}".format(
+                virustotal['positives'],
+                virustotal['total']
+            ), 
+            "types": "comment",
+            "categories": category, 
+            "comment": comment
+        }) 
+    else: 
+        r["results"].append({ 
+            "values": "Sample not detected on VirusTotal", 
+            "types": "comment",
+            "categories": category, 
+            "comment": comment
+        })
+    
+
+def processNetwork(r, network):
+    category = "Network activity"
+    
+    for host in network['hosts']:
+        r["results"].append({ 
+            "values": host['ip'], 
+            "types": "ip-dst",
+            "categories": category,  
+            "comment": "Cuckoo analysis: Observed network traffic"
+        })
+    
+
+def processBinary(r, target, initial = False, dropped = False):
+    if(initial): 
+        comment = "Cuckoo analysis: Initial file"
+        category = "Payload delivery"
+    elif(dropped):
+        category = "Artifacts dropped"
+        comment = "Cuckoo analysis: Dropped file"
+    
+    r["results"].append({ 
+        "values": target['name'], 
+        "types": "filename",
+        "categories": category, 
+        "comment": comment
+    })
+    
+    r["results"].append({ 
+        "values": target['md5'], 
+        "types": "md5",
+        "categories": category, 
+        "comment": comment
+    })
+    
+    r["results"].append({ 
+        "values": target['sha1'], 
+        "types": "sha1",
+        "categories": category, 
+        "comment": comment
+    })
+    
+    r["results"].append({ 
+        "values": target['sha256'], 
+        "types": "sha256",
+        "categories": category, 
+        "comment": comment
+    })
+    
+    r["results"].append({ 
+        "values": target['sha512'], 
+        "types": "sha512",
+        "categories": category, 
+        "comment": comment
+    })
+    
+    # todo : add file size?
+    
+    if(target.get('guest_paths')):
+        r["results"].append({ 
+            "values": target['guest_paths'],
+            "types": "filename",
+            "categories": "Payload installation", 
+            "comment": comment + " - Path"
+        })
+    
+
+def introspection():
+    modulesetup = {}
+    try:
+        userConfig
+        modulesetup['userConfig'] = userConfig
+    except NameError:
+        pass
+    try:
+        inputSource
+        modulesetup['inputSource'] = inputSource
+    except NameError:
+        pass
+    return modulesetup
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo
+
+if __name__ == '__main__':
+    x = open('test.json', 'r')
+    q = []
+    q['data'] = x.read()
+    q = base64.base64encode(q)
+    
+    handler(q)

From 4a8ccb54fba29736a3600677b28814d41d7d6f98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 15 Dec 2016 16:49:56 +0100
Subject: [PATCH 47/54] Refactoring of domaintools expansion module

---
 misp_modules/modules/expansion/domaintools.py | 159 ++++++++++++------
 1 file changed, 105 insertions(+), 54 deletions(-)

diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py
index 6382726..8af89ed 100755
--- a/misp_modules/modules/expansion/domaintools.py
+++ b/misp_modules/modules/expansion/domaintools.py
@@ -15,9 +15,10 @@ log.addHandler(ch)
 
 misperrors = {'error': 'Error'}
 mispattributes = {
-    'input': ['domain'],
+    'input': ['domain', 'email-src', 'email-dst', 'target-email', 'whois-registrant-email',
+              'whois-registrant-name', 'whois-registrant-phone', 'ip-src', 'ip-dst'],
     'output': ['whois-registrant-email', 'whois-registrant-phone', 'whois-registrant-name',
-               'whois-registrar', 'whois-creation-date', 'freetext']
+               'whois-registrar', 'whois-creation-date', 'freetext', 'domain']
 }
 
 moduleinfo = {
@@ -29,6 +30,12 @@ moduleinfo = {
 
 moduleconfig = ['username', 'api_key']
 
+query_profiles = [
+    {'inputs': ['domain'], 'services': ['parsed_whois', 'domain_profile', 'reputation', 'reverse_ip']},
+    {'inputs': ['email-src', 'email-dst', 'target-email', 'whois-registrant-email', 'whois-registrant-name', 'whois-registrant-phone'], 'services': ['reverse_whois']},
+    {'inputs': ['ip-src', 'ip-dst'], 'services': ['host_domains', 'reverse_ip_whois']}
+]
+
 
 class DomainTools(object):
 
@@ -39,6 +46,7 @@ class DomainTools(object):
         self.registrar = {}
         self.creation_date = {}
         self.domain_ip = {}
+        self.domain = {}
         self.risk = ()
         self.freetext = ''
 
@@ -68,6 +76,9 @@ class DomainTools(object):
     def add_ip(self, ip, comment=None):
         self.domain_ip = self._add_value(self.domain_ip, ip, comment)
 
+    def add_domain(self, domain, comment=None):
+        self.domain = self._add_value(self.domain, domain, comment)
+
     def dump(self):
         to_return = []
         if self.reg_mail:
@@ -88,6 +99,9 @@ class DomainTools(object):
         if self.domain_ip:
             for ip, comment in self.domain_ip.items():
                 to_return.append({'types': ['ip-dst', 'ip-src'], 'values': [ip], 'comment': comment or ''})
+        if self.domain:
+            for domain, comment in self.domain.items():
+                to_return.append({'type': 'domain', 'values': [domain], 'comment': comment or ''})
         if self.freetext:
             to_return.append({'type': 'freetext', 'values': [self.freetext], 'comment': 'Freetext import'})
         if self.risk:
@@ -95,65 +109,14 @@ class DomainTools(object):
         return to_return
 
 
-def handler(q=False):
-    if not q:
-        return q
-
-    request = json.loads(q)
-    to_query = None
-    for t in mispattributes['input']:
-        to_query = request.get(t)
-        if to_query:
-            break
-    if not to_query:
-        misperrors['error'] = "Unsupported attributes type"
-        return misperrors
-
-    if request.get('config'):
-        if (request['config'].get('username') is None) or (request['config'].get('api_key') is None):
-            misperrors['error'] = 'DomainTools authentication is incomplete'
-            return misperrors
-        else:
-            domtools = API(request['config'].get('username'), request['config'].get('api_key'))
-    else:
-        misperrors['error'] = 'DomainTools authentication is missing'
-        return misperrors
-
+def parsed_whois(domtools, to_query, values):
     whois_entry = domtools.parsed_whois(to_query)
-    profile = domtools.domain_profile(to_query)
-    # NOTE: profile['website_data']['response_code'] could be used to see if the host is still up. Maybe set a tag.
-    reputation = domtools.reputation(to_query, include_reasons=True)
-    # NOTE: use that value in a tag when we will have attribute level tagging
-    values = DomainTools()
-
     if whois_entry.get('error'):
         misperrors['error'] = whois_entry['error']['message']
         return misperrors
 
-    if profile.get('error'):
-        misperrors['error'] = profile['error']['message']
-        return misperrors
-
-    if reputation and not reputation.get('error'):
-        reasons = ', '.join(reputation['reasons'])
-        values.risk = [reputation['risk_score'], 'Risk value of {} (via Domain Tools), Reasons: {}'.format(to_query, reasons)]
-
     if whois_entry.get('registrant'):
         values.add_name(whois_entry['registrant'], 'Parsed registrant')
-    if profile.get('registrant'):
-        values.add_name(profile['registrant']['name'], 'Profile registrant')
-
-    if profile.get('server'):
-        other_domains = profile['server']['other_domains']
-        values.add_ip(profile['server']['ip_address'], 'IP of {} (via DomainTools). Has {} other domains.'.format(to_query, other_domains))
-
-    if profile.get('registration'):
-        if profile['registration'].get('created'):
-            values.add_creation_date(profile['registration']['created'], 'created')
-        if profile['registration'].get('updated'):
-            values.add_creation_date(profile['registration']['updated'], 'updated')
-        if profile['registration'].get('registrar'):
-            values.add_registrar(profile['registration']['registrar'], 'name')
 
     if whois_entry.get('registration'):
         values.add_creation_date(whois_entry['registration']['created'], 'timestamp')
@@ -180,6 +143,94 @@ def handler(q=False):
         for mail in whois_entry.emails():
             if mail not in values.reg_mail.keys():
                 values.add_mail(mail, 'Maybe registrar')
+    return values
+
+
+def domain_profile(domtools, to_query, values):
+    profile = domtools.domain_profile(to_query)
+    # NOTE: profile['website_data']['response_code'] could be used to see if the host is still up. Maybe set a tag.
+    if profile.get('error'):
+        misperrors['error'] = profile['error']['message']
+        return misperrors
+
+    if profile.get('registrant'):
+        values.add_name(profile['registrant']['name'], 'Profile registrant')
+
+    if profile.get('server'):
+        other_domains = profile['server']['other_domains']
+        values.add_ip(profile['server']['ip_address'], 'IP of {} (via DomainTools). Has {} other domains.'.format(to_query, other_domains))
+
+    if profile.get('registration'):
+        if profile['registration'].get('created'):
+            values.add_creation_date(profile['registration']['created'], 'created')
+        if profile['registration'].get('updated'):
+            values.add_creation_date(profile['registration']['updated'], 'updated')
+        if profile['registration'].get('registrar'):
+            values.add_registrar(profile['registration']['registrar'], 'name')
+    return values
+
+
+def reputation(domtools, to_query, values):
+    rep = domtools.reputation(to_query, include_reasons=True)
+    # NOTE: use that value in a tag when we will have attribute level tagging
+
+    if rep and not rep.get('error'):
+        reasons = ', '.join(rep['reasons'])
+        values.risk = [rep['risk_score'], 'Risk value of {} (via Domain Tools), Reasons: {}'.format(to_query, reasons)]
+
+    return values
+
+
+def reverse_ip(domtools, to_query, values):
+    rev_ip = domtools.reverse_ip(to_query)
+    if rev_ip and not rev_ip.get('error'):
+        ip_addresses = rev_ip['ip_addresses']
+        values.add_ip(ip_addresses['ip_address'], 'IP of {} (via DomainTools). Has {} other domains.'.format(to_query, ip_addresses['domain_count']))
+        for d in ip_addresses['domain_names']:
+            values.add_domain(d, 'Other domain on {}.'.format(ip_addresses['ip_address']))
+    return values
+
+
+def get_services(request):
+    for t in mispattributes['input']:
+        to_query = request.get(t)
+        if not to_query:
+            continue
+        for p in query_profiles:
+            if t in p['inputs']:
+                return p['services']
+
+
+def handler(q=False):
+    if not q:
+        return q
+
+    request = json.loads(q)
+    to_query = None
+    for t in mispattributes['input']:
+        to_query = request.get(t)
+        if to_query:
+            break
+    if not to_query:
+        misperrors['error'] = "Unsupported attributes type"
+        return misperrors
+
+    if request.get('config'):
+        if (request['config'].get('username') is None) or (request['config'].get('api_key') is None):
+            misperrors['error'] = 'DomainTools authentication is incomplete'
+            return misperrors
+        else:
+            domtools = API(request['config'].get('username'), request['config'].get('api_key'))
+    else:
+        misperrors['error'] = 'DomainTools authentication is missing'
+        return misperrors
+
+    values = DomainTools()
+    services = get_services(request)
+    if services:
+        for s in services:
+            globals()[s](domtools, to_query, values)
+
     return {'results': values.dump()}
 
 

From 467e50327d773b3e5eb4ebf7b78e20528dfc5528 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Fri, 16 Dec 2016 11:22:22 +0100
Subject: [PATCH 48/54] Add reverse lookup

---
 misp_modules/modules/expansion/domaintools.py | 39 +++++++++++++++++--
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py
index 8af89ed..cd35863 100755
--- a/misp_modules/modules/expansion/domaintools.py
+++ b/misp_modules/modules/expansion/domaintools.py
@@ -33,7 +33,7 @@ moduleconfig = ['username', 'api_key']
 query_profiles = [
     {'inputs': ['domain'], 'services': ['parsed_whois', 'domain_profile', 'reputation', 'reverse_ip']},
     {'inputs': ['email-src', 'email-dst', 'target-email', 'whois-registrant-email', 'whois-registrant-name', 'whois-registrant-phone'], 'services': ['reverse_whois']},
-    {'inputs': ['ip-src', 'ip-dst'], 'services': ['host_domains', 'reverse_ip_whois']}
+    {'inputs': ['ip-src', 'ip-dst'], 'services': ['host_domains']}
 ]
 
 
@@ -173,11 +173,9 @@ def domain_profile(domtools, to_query, values):
 def reputation(domtools, to_query, values):
     rep = domtools.reputation(to_query, include_reasons=True)
     # NOTE: use that value in a tag when we will have attribute level tagging
-
     if rep and not rep.get('error'):
         reasons = ', '.join(rep['reasons'])
         values.risk = [rep['risk_score'], 'Risk value of {} (via Domain Tools), Reasons: {}'.format(to_query, reasons)]
-
     return values
 
 
@@ -191,6 +189,41 @@ def reverse_ip(domtools, to_query, values):
     return values
 
 
+def reverse_whois(domtools, to_query, values):
+    rev_whois = domtools.reverse_whois(to_query, mode='purchase')
+    if rev_whois.get('error'):
+        misperrors['error'] = rev_whois['error']['message']
+        return misperrors
+    for d in rev_whois['domains']:
+        values.add_domain(d, 'Reverse domain related to {}.'.format(to_query))
+    return values
+
+
+def host_domains(domtools, to_query, values):
+    hostdom = domtools.host_domains(to_query)
+    if hostdom.get('error'):
+        misperrors['error'] = hostdom['error']['message']
+        return misperrors
+    ip_addresses = hostdom['ip_addresses']
+    if to_query != ip_addresses['ip_address']:
+        values.add_ip(ip_addresses['ip_address'], 'IP of {} (via DomainTools). Has {} other domains.'.format(to_query, ip_addresses['domain_count']))
+    for d in ip_addresses['domain_names']:
+        values.add_domain(d, 'Other domain on {}.'.format(ip_addresses['ip_address']))
+    return values
+
+
+def reverse_ip_whois(domtools, to_query, values):
+    # Disabled for now, dies with domaintools.exceptions.NotAuthorizedException
+    rev_whois = domtools.reverse_ip_whois(ip=to_query)
+    print(rev_whois)
+    if rev_whois.get('error'):
+        misperrors['error'] = rev_whois['error']['message']
+        return misperrors
+    # for d in rev_whois['domains']:
+    #    values.add_domain(d, 'Reverse domain related to {}.'.format(to_query))
+    return values
+
+
 def get_services(request):
     for t in mispattributes['input']:
         to_query = request.get(t)

From ffc0a97126fdd60ba7eef5efea0b2adbf7ac6fd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Fri, 16 Dec 2016 11:52:51 +0100
Subject: [PATCH 49/54] Catch exception

---
 misp_modules/modules/expansion/domaintools.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py
index cd35863..724da5d 100755
--- a/misp_modules/modules/expansion/domaintools.py
+++ b/misp_modules/modules/expansion/domaintools.py
@@ -261,8 +261,11 @@ def handler(q=False):
     values = DomainTools()
     services = get_services(request)
     if services:
-        for s in services:
-            globals()[s](domtools, to_query, values)
+        try:
+            for s in services:
+                globals()[s](domtools, to_query, values)
+        except Exception as e:
+            print(type(e), e)
 
     return {'results': values.dump()}
 

From 60d3e0a1acc0b1511984f32091fefaf1b8263b66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Fri, 16 Dec 2016 12:02:28 +0100
Subject: [PATCH 50/54] Better error reporting

---
 misp_modules/modules/expansion/domaintools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py
index 724da5d..d952fdf 100755
--- a/misp_modules/modules/expansion/domaintools.py
+++ b/misp_modules/modules/expansion/domaintools.py
@@ -265,7 +265,7 @@ def handler(q=False):
             for s in services:
                 globals()[s](domtools, to_query, values)
         except Exception as e:
-            print(type(e), e)
+            print(to_query, type(e), e)
 
     return {'results': values.dump()}
 

From 064c3e364930f868ef08c067b99611e2f8bb3bd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Fri, 16 Dec 2016 15:14:48 +0100
Subject: [PATCH 51/54] Fix path to config file

---
 .../modules/expansion/geoip_country.py        | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py
index f5a1984..047487d 100644
--- a/misp_modules/modules/expansion/geoip_country.py
+++ b/misp_modules/modules/expansion/geoip_country.py
@@ -1,5 +1,8 @@
-import json, pygeoip
-import sys, logging
+import json
+import pygeoip
+import sys
+import os
+import logging
 import configparser
 
 log = logging.getLogger('geoip_country')
@@ -20,9 +23,10 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
 
 # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
 config = configparser.ConfigParser()
-config.read('geoip_country.cfg')
+config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'geoip_country.cfg'))
 gi = pygeoip.GeoIP(config.get('GEOIP', 'database'))
 
+
 def handler(q=False):
     if q is False:
         return False
@@ -35,7 +39,7 @@ def handler(q=False):
     elif request.get('domain|ip'):
         toquery = request['domain|ip'].split('|')[1]
     else:
-        return false
+        return False
 
     log.debug(toquery)
 
@@ -45,14 +49,15 @@ def handler(q=False):
         misperrors['error'] = "GeoIP resolving error"
         return misperrors
 
-    r = {'results': [{'types': mispattributes['output'],
-                      'values': [str(answer)]}]}
+    r = {'results': [{'types': mispattributes['output'], 'values': [str(answer)]}]}
 
     return r
 
+
 def introspection():
     return mispattributes
 
+
 def version():
-    moduleinfo['config'] = moduleconfig
+    # moduleinfo['config'] = moduleconfig
     return moduleinfo

From 9bf1c936cf486a6242465dc6b763120947ef6eb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Fri, 16 Dec 2016 15:22:16 +0100
Subject: [PATCH 52/54] Do not crash if the dat file is not available

---
 misp_modules/modules/expansion/geoip_country.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py
index 047487d..31c1b6a 100644
--- a/misp_modules/modules/expansion/geoip_country.py
+++ b/misp_modules/modules/expansion/geoip_country.py
@@ -21,10 +21,14 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann',
               'description': 'Query a local copy of Maxminds Geolite database',
               'module-type': ['expansion', 'hover']}
 
-# get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
-config = configparser.ConfigParser()
-config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'geoip_country.cfg'))
-gi = pygeoip.GeoIP(config.get('GEOIP', 'database'))
+try:
+    # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
+    config = configparser.ConfigParser()
+    config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'geoip_country.cfg'))
+    gi = pygeoip.GeoIP(config.get('GEOIP', 'database'))
+    enabled = True
+except:
+    enabled = False
 
 
 def handler(q=False):

From 1753c89bed0c1c0270872c25fd567358ad376334 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Sat, 17 Dec 2016 15:06:08 +0100
Subject: [PATCH 53/54] GeoIP module added

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index e7da93d..8e53c4d 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
 * [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
 * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
+* [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
 * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
 * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
 * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.

From 97c49e267577624dbfbe072d2c4339a87be18c4a Mon Sep 17 00:00:00 2001
From: "robin.marsollier@conix.fr" <robin.marsollier@conix.fr>
Date: Tue, 20 Dec 2016 16:14:08 +0100
Subject: [PATCH 54/54] add libjpeg-dev as a dep to allow pillow to be
 installed succesfully

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 8e53c4d..77a320d 100644
--- a/README.md
+++ b/README.md
@@ -44,7 +44,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 ## How to install and start MISP modules?
 
 ~~~~bash
-sudo apt-get install python3-dev python3-pip libpq5
+sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev
 cd /usr/local/src/
 sudo git clone https://github.com/MISP/misp-modules.git
 cd misp-modules