Multiple clanges in the vmray modules.

* Generic fix to load modules requiring a local library
* Fix python3 support
* PEP8 related cleanups

misp_modules/__init__.py

@@ -123,6 +123,8 @@ def load_modules(mod_dir):
         if os.path.basename(root).startswith("."):
             continue
         for filename in fnmatch.filter(filenames, '*.py'):
+            if root.split('/')[-1].startswith('_'):
+                continue
             if filename == '__init__.py':
                 continue
             modulename = filename.split(".")[0]
@@ -145,7 +147,7 @@ def load_package_modules():
     mhandlers = {}
     modules = []
     for path, module in sys.modules.items():
-        r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
+        r = re.findall("misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
         if r and len(r[0]) == 2:
             moduletype, modulename = r[0]
             mhandlers[modulename] = module

misp_modules/modules/expansion/__init__.py

@@ -1,2 +1,4 @@
-__all__ = ['vmray_submit','asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
+from . import _vmray
+
+__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
            'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'wiki']

misp_modules/modules/expansion/_vmray/vmray_rest_api.py

@@ -1,16 +1,12 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 """Python client library for VMRay REST API"""
 
 import base64
 import datetime
 import os.path
 import requests
-#import urlparse
 import urllib.parse
 
-from io import IOBase
-from io import BytesIO
-
 # disable nasty certification warning
 # pylint: disable=no-member
 try:
@@ -87,7 +83,7 @@ class VMRayRESTAPI(object):
                     filename = value["filename"]
                     sample = value["data"]
                     file_params[key] = (filename, sample, "application/octet-stream")
-                elif isinstance(value, file) or hasattr(value, "read"):
+                elif hasattr(value, "read"):
                     filename = os.path.split(value.name)[1]
                     # For the following block refer to DEV-1820
                     try:

misp_modules/modules/expansion/vmray_submit.py

@@ -12,16 +12,12 @@ TODO:
 '''
 
 import json
-import re
 import base64
 
-import sys
-import os
-base_dir = os.path.dirname(__file__) or '.'
-sys.path.append(base_dir)
-from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError
 import io
 
+from ._vmray.vmray_rest_api import VMRayRESTAPI
+
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['attachment'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']}
 moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe',
@@ -81,7 +77,7 @@ def handler(q=False):
     if data and attachment:
         args = {}
         args["shareable"] = shareable
-        args["sample_file"] = {'data': io.BytesIO( data ) , 'filename': attachment }
+        args["sample_file"] = {'data': io.BytesIO(data), 'filename': attachment}
         args["reanalyze"] = reanalyze
 
         try:
@@ -118,13 +114,13 @@ def vmrayProcess(vmraydata):
             # Result received?
             if submissions and jobs:
                 r = {'results': []}
-                r["results"].append( {"types": "md5", "values": submissions["submission_sample_md5"]} )
+                r["results"].append({"types": "md5", "values": submissions["submission_sample_md5"]})
-                r["results"].append( {"types": "sha1", "values": submissions["submission_sample_sha1"]} )
+                r["results"].append({"types": "sha1", "values": submissions["submission_sample_sha1"]})
-                r["results"].append( {"types": "sha256", "values": submissions["submission_sample_sha256"]} )
+                r["results"].append({"types": "sha256", "values": submissions["submission_sample_sha256"]})
-                r["results"].append( {"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]} )
+                r["results"].append({"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]})
-                r["results"].append( {"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]} )
+                r["results"].append({"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]})
-                r["results"].append( {"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]} )
+                r["results"].append({"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]})
-                r["results"].append( {"types": "link", "values": submissions["submission_webif_url"]} )
+                r["results"].append({"types": "link", "values": submissions["submission_webif_url"]})
 
                 # Include data from different jobs
                 if include_vmrayjobids:
@@ -132,7 +128,7 @@ def vmrayProcess(vmraydata):
                         job_id = job["job_id"]
                         job_vm_name = job["job_vm_name"]
                         job_configuration_name = job["job_configuration_name"]
-                        r["results"].append( {"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name) })
+                        r["results"].append({"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name)})
                 return r
             else:
                 misperrors['error'] = "No valid results returned."
@@ -149,4 +145,3 @@ def vmraySubmit(api, args):
     ''' Submit the sample to VMRay'''
     vmraydata = api.call("POST", "/rest/sample/submit", args)
     return vmraydata
-

misp_modules/modules/import_mod/__init__.py

@@ -1 +1,3 @@
-__all__ = ['vmray_import','testimport', 'ocr', 'stiximport']
+from . import _vmray
+
+__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport']

misp_modules/modules/import_mod/_vmray/vmray_rest_api.py

@@ -5,12 +5,8 @@ import base64
 import datetime
 import os.path
 import requests
-#import urlparse
 import urllib.parse
 
-from io import IOBase
-from io import BytesIO
-
 # disable nasty certification warning
 # pylint: disable=no-member
 try:
@@ -87,7 +83,7 @@ class VMRayRESTAPI(object):
                     filename = value["filename"]
                     sample = value["data"]
                     file_params[key] = (filename, sample, "application/octet-stream")
-                elif isinstance(value, file) or hasattr(value, "read"):
+                elif hasattr(value, "read"):
                     filename = os.path.split(value.name)[1]
                     # For the following block refer to DEV-1820
                     try:

misp_modules/modules/import_mod/vmray_import.py

@@ -16,42 +16,36 @@ TODO:
 
 import json
 import re
-import sys
+
-import os
+from ._vmray.vmray_rest_api import VMRayRESTAPI
-base_dir = os.path.dirname(__file__) or '.'
-sys.path.append(base_dir)
-from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError
 
 misperrors = {'error': 'Error'}
 inputSource = []
 moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe',
               'description': 'Import VMRay (VTI) results',
               'module-type': ['import']}
-userConfig = {
+userConfig = {'include_textdescr': {'type': 'Boolean',
-               'include_textdescr': {
+                                    'message': 'Include textual description'
-                 'type': 'Boolean',
+                                    },
-                 'message': 'Include textual description'
+              'include_analysisid': {'type': 'Boolean',
-               },
+                                     'message': 'Include VMRay analysis_id text'
-               'include_analysisid': {
+                                     },
-                 'type': 'Boolean',
+              'only_network_info': {'type': 'Boolean',
-                 'message': 'Include VMRay analysis_id text'
+                                    'message': 'Only include network (src-ip, hostname, domain, ...) information'
-               },
+                                    },
-               'only_network_info': {
+              'sample_id': {'type': 'Integer',
-                 'type': 'Boolean',
+                            'errorMessage': 'Expected a sample ID',
-                 'message': 'Only include network (src-ip, hostname, domain, ...) information'
+                            'message': 'The VMRay sample_id'
-               },
+                            }
-               'sample_id': {
+              }
-                 'type': 'Integer',
+
-                 'errorMessage': 'Expected a sample ID',
-                 'message': 'The VMRay sample_id'
-               }
-             };
 moduleconfig = ['apikey', 'url']
 
 include_textdescr = False
 include_analysisid = False
 only_network_info = False
 
+
 def handler(q=False):
     global include_textdescr
     global include_analysisid
@@ -104,8 +98,8 @@ def handler(q=False):
                                     url1 = "https://cloud.vmray.com/user/analysis/view?from_sample_id=%u" % sample_id
                                     url2 = "&id=%u" % analysis_id
                                     url3 = "&sub=%2Freport%2Foverview.html"
-                                    p["results"].append({ "values": url1 + url2 + url3, "types": "link" })
+                                    p["results"].append({"values": url1 + url2 + url3, "types": "link"})
-                                vmray_results = {'results': vmray_results["results"] + p["results"] }
+                                vmray_results = {'results': vmray_results["results"] + p["results"]}
 
                 # Clean up (remove doubles)
                 vmray_results = vmrayCleanup(vmray_results)
@@ -121,7 +115,6 @@ def handler(q=False):
         return misperrors
 
 
-
 def introspection():
     modulesetup = {}
     try:
@@ -175,42 +168,42 @@ def vmrayVtiPatterns(vti_patterns):
             elif pattern["category"] == "_network" and pattern["operation"] == "_connect":
                 content = vmrayConnect(pattern)
 
-            elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page":
+            elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page":
                 content = vmrayGeneric(pattern)
-            elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint":
+            elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint":
                 content = vmrayGeneric(pattern, "mutex", 1)
 
-            elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
+            elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
                 content = vmrayGeneric(pattern)
-            elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage":
+            elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage":
                 content = vmrayGeneric(pattern)
 
-            elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file":
+            elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file":
                 content = vmrayGeneric(pattern, "filename", 1)
-            elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file":
+            elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file":
                 content = vmrayGeneric(pattern, "filename", 1)
 
-            elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory":
+            elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory":
                 content = vmrayGeneric(pattern)
-            elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow":
+            elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow":
                 content = vmrayGeneric(pattern)
-            elif only_network_info == False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files":
+            elif only_network_info is False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files":
                 content = vmrayGeneric(pattern)
 
-            elif only_network_info == False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script":
+            elif only_network_info is False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script":
                 content = vmrayGeneric(pattern, "regkey", 1)
-            elif only_network_info == False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges":
+            elif only_network_info is False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges":
                 content = vmrayGeneric(pattern)
 
             if content:
-                r["results"].append( content["attributes"] )
+                r["results"].append(content["attributes"])
-                r["results"].append( content["text"] )
+                r["results"].append(content["text"])
 
         # Remove empty results
-        r["results"] = [x for x in r["results"] if isinstance(x, dict) and  len(x["values"]) != 0]
+        r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0]
         for el in r["results"]:
-            if not el in y["results"]:
+            if el not in y["results"]:
-                y["results"].append( el )
+                y["results"].append(el)
         return y
     else:
         return False
@@ -221,22 +214,22 @@ def vmrayCleanup(x):
     y = {'results': []}
 
     for el in x["results"]:
-        if not el in y["results"]:
+        if el not in y["results"]:
-            y["results"].append( el )
+            y["results"].append(el)
     return y
 
 
 def vmraySanitizeInput(s):
     ''' Sanitize some input so it gets properly imported in MISP'''
     if s:
-        s = s.replace('"','')
+        s = s.replace('"', '')
-        s =  re.sub('\\\\', r'\\', s)
+        s = re.sub('\\\\', r'\\', s)
         return s
     else:
         return False
 
 
-def vmrayGeneric(el, attr = "", attrpos = 1):
+def vmrayGeneric(el, attr="", attrpos=1):
     ''' Convert a 'generic' VTI pattern to MISP data'''
 
     r = {"values": []}
@@ -257,8 +250,7 @@ def vmrayGeneric(el, attr = "", attrpos = 1):
                 f["values"].append(vmraySanitizeInput(content))
                 f["types"] = ["text"]
 
-            return {    "text": f,
+            return {"text": f, "attributes": r}
-                        "attributes": r}
         else:
             return False
     else:
@@ -276,7 +268,7 @@ def vmrayConnect(el):
         content = el["technique_desc"]
         if content:
             target = content.split("\"")
-            port = (target[1].split(":"))[1]
+            # port = (target[1].split(":"))[1]  ## FIXME: not used
             host = (target[1].split(":"))[0]
             if ipre.match(str(host)):
                 r["values"].append(host)
@@ -292,8 +284,7 @@ def vmrayConnect(el):
                 f["values"].append(vmraySanitizeInput(content))
                 f["types"] = ["text"]
 
-            return {    "text": f,
+            return {"text": f, "attributes": r}
-                        "attributes": r}
         else:
             return False
     else: