From c676587461b0ca102449ee474f18c28fd935b79d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 15 Nov 2016 16:43:11 +0100 Subject: [PATCH] Multiple clanges in the vmray modules. * Generic fix to load modules requiring a local library * Fix python3 support * PEP8 related cleanups --- misp_modules/__init__.py | 4 +- misp_modules/modules/expansion/__init__.py | 4 +- .../modules/expansion/_vmray/__init__.py | 0 .../_vmray}/vmray_rest_api.py | 8 +- .../modules/expansion/vmray_submit.py | 27 +++--- misp_modules/modules/import_mod/__init__.py | 4 +- .../modules/import_mod/_vmray/__init__.py | 0 .../_vmray}/vmray_rest_api.py | 6 +- .../modules/import_mod/vmray_import.py | 97 +++++++++---------- 9 files changed, 67 insertions(+), 83 deletions(-) create mode 100644 misp_modules/modules/expansion/_vmray/__init__.py rename misp_modules/modules/{import_mod => expansion/_vmray}/vmray_rest_api.py (97%) create mode 100644 misp_modules/modules/import_mod/_vmray/__init__.py rename misp_modules/modules/{expansion => import_mod/_vmray}/vmray_rest_api.py (97%) diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index a2562c4..1edfc86 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -123,6 +123,8 @@ def load_modules(mod_dir): if os.path.basename(root).startswith("."): continue for filename in fnmatch.filter(filenames, '*.py'): + if root.split('/')[-1].startswith('_'): + continue if filename == '__init__.py': continue modulename = filename.split(".")[0] @@ -145,7 +147,7 @@ def load_package_modules(): mhandlers = {} modules = [] for path, module in sys.modules.items(): - r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path) + r = re.findall("misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path) if r and len(r[0]) == 2: moduletype, modulename = r[0] mhandlers[modulename] = module diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index fdbdfb8..62a6ecd 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,2 +1,4 @@ -__all__ = ['vmray_submit','asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', +from . import _vmray + +__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'wiki'] diff --git a/misp_modules/modules/expansion/_vmray/__init__.py b/misp_modules/modules/expansion/_vmray/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/misp_modules/modules/import_mod/vmray_rest_api.py b/misp_modules/modules/expansion/_vmray/vmray_rest_api.py similarity index 97% rename from misp_modules/modules/import_mod/vmray_rest_api.py rename to misp_modules/modules/expansion/_vmray/vmray_rest_api.py index 64b8024..4d5245b 100644 --- a/misp_modules/modules/import_mod/vmray_rest_api.py +++ b/misp_modules/modules/expansion/_vmray/vmray_rest_api.py @@ -1,16 +1,12 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 """Python client library for VMRay REST API""" import base64 import datetime import os.path import requests -#import urlparse import urllib.parse -from io import IOBase -from io import BytesIO - # disable nasty certification warning # pylint: disable=no-member try: @@ -87,7 +83,7 @@ class VMRayRESTAPI(object): filename = value["filename"] sample = value["data"] file_params[key] = (filename, sample, "application/octet-stream") - elif isinstance(value, file) or hasattr(value, "read"): + elif hasattr(value, "read"): filename = os.path.split(value.name)[1] # For the following block refer to DEV-1820 try: diff --git a/misp_modules/modules/expansion/vmray_submit.py b/misp_modules/modules/expansion/vmray_submit.py index 9ab4e05..407853c 100644 --- a/misp_modules/modules/expansion/vmray_submit.py +++ b/misp_modules/modules/expansion/vmray_submit.py @@ -12,16 +12,12 @@ TODO: ''' import json -import re import base64 -import sys -import os -base_dir = os.path.dirname(__file__) or '.' -sys.path.append(base_dir) -from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError import io +from ._vmray.vmray_rest_api import VMRayRESTAPI + misperrors = {'error': 'Error'} mispattributes = {'input': ['attachment'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']} moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe', @@ -81,7 +77,7 @@ def handler(q=False): if data and attachment: args = {} args["shareable"] = shareable - args["sample_file"] = {'data': io.BytesIO( data ) , 'filename': attachment } + args["sample_file"] = {'data': io.BytesIO(data), 'filename': attachment} args["reanalyze"] = reanalyze try: @@ -118,13 +114,13 @@ def vmrayProcess(vmraydata): # Result received? if submissions and jobs: r = {'results': []} - r["results"].append( {"types": "md5", "values": submissions["submission_sample_md5"]} ) - r["results"].append( {"types": "sha1", "values": submissions["submission_sample_sha1"]} ) - r["results"].append( {"types": "sha256", "values": submissions["submission_sample_sha256"]} ) - r["results"].append( {"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]} ) - r["results"].append( {"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]} ) - r["results"].append( {"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]} ) - r["results"].append( {"types": "link", "values": submissions["submission_webif_url"]} ) + r["results"].append({"types": "md5", "values": submissions["submission_sample_md5"]}) + r["results"].append({"types": "sha1", "values": submissions["submission_sample_sha1"]}) + r["results"].append({"types": "sha256", "values": submissions["submission_sample_sha256"]}) + r["results"].append({"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]}) + r["results"].append({"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]}) + r["results"].append({"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]}) + r["results"].append({"types": "link", "values": submissions["submission_webif_url"]}) # Include data from different jobs if include_vmrayjobids: @@ -132,7 +128,7 @@ def vmrayProcess(vmraydata): job_id = job["job_id"] job_vm_name = job["job_vm_name"] job_configuration_name = job["job_configuration_name"] - r["results"].append( {"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name) }) + r["results"].append({"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name)}) return r else: misperrors['error'] = "No valid results returned." @@ -149,4 +145,3 @@ def vmraySubmit(api, args): ''' Submit the sample to VMRay''' vmraydata = api.call("POST", "/rest/sample/submit", args) return vmraydata - diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index 4b30a1a..70b674a 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1 +1,3 @@ -__all__ = ['vmray_import','testimport', 'ocr', 'stiximport'] +from . import _vmray + +__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport'] diff --git a/misp_modules/modules/import_mod/_vmray/__init__.py b/misp_modules/modules/import_mod/_vmray/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/misp_modules/modules/expansion/vmray_rest_api.py b/misp_modules/modules/import_mod/_vmray/vmray_rest_api.py similarity index 97% rename from misp_modules/modules/expansion/vmray_rest_api.py rename to misp_modules/modules/import_mod/_vmray/vmray_rest_api.py index 64b8024..d37c6f2 100644 --- a/misp_modules/modules/expansion/vmray_rest_api.py +++ b/misp_modules/modules/import_mod/_vmray/vmray_rest_api.py @@ -5,12 +5,8 @@ import base64 import datetime import os.path import requests -#import urlparse import urllib.parse -from io import IOBase -from io import BytesIO - # disable nasty certification warning # pylint: disable=no-member try: @@ -87,7 +83,7 @@ class VMRayRESTAPI(object): filename = value["filename"] sample = value["data"] file_params[key] = (filename, sample, "application/octet-stream") - elif isinstance(value, file) or hasattr(value, "read"): + elif hasattr(value, "read"): filename = os.path.split(value.name)[1] # For the following block refer to DEV-1820 try: diff --git a/misp_modules/modules/import_mod/vmray_import.py b/misp_modules/modules/import_mod/vmray_import.py index 397bd39..dceeb1f 100644 --- a/misp_modules/modules/import_mod/vmray_import.py +++ b/misp_modules/modules/import_mod/vmray_import.py @@ -16,42 +16,36 @@ TODO: import json import re -import sys -import os -base_dir = os.path.dirname(__file__) or '.' -sys.path.append(base_dir) -from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError + +from ._vmray.vmray_rest_api import VMRayRESTAPI misperrors = {'error': 'Error'} inputSource = [] moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe', 'description': 'Import VMRay (VTI) results', 'module-type': ['import']} -userConfig = { - 'include_textdescr': { - 'type': 'Boolean', - 'message': 'Include textual description' - }, - 'include_analysisid': { - 'type': 'Boolean', - 'message': 'Include VMRay analysis_id text' - }, - 'only_network_info': { - 'type': 'Boolean', - 'message': 'Only include network (src-ip, hostname, domain, ...) information' - }, - 'sample_id': { - 'type': 'Integer', - 'errorMessage': 'Expected a sample ID', - 'message': 'The VMRay sample_id' - } - }; +userConfig = {'include_textdescr': {'type': 'Boolean', + 'message': 'Include textual description' + }, + 'include_analysisid': {'type': 'Boolean', + 'message': 'Include VMRay analysis_id text' + }, + 'only_network_info': {'type': 'Boolean', + 'message': 'Only include network (src-ip, hostname, domain, ...) information' + }, + 'sample_id': {'type': 'Integer', + 'errorMessage': 'Expected a sample ID', + 'message': 'The VMRay sample_id' + } + } + moduleconfig = ['apikey', 'url'] include_textdescr = False include_analysisid = False only_network_info = False + def handler(q=False): global include_textdescr global include_analysisid @@ -104,8 +98,8 @@ def handler(q=False): url1 = "https://cloud.vmray.com/user/analysis/view?from_sample_id=%u" % sample_id url2 = "&id=%u" % analysis_id url3 = "&sub=%2Freport%2Foverview.html" - p["results"].append({ "values": url1 + url2 + url3, "types": "link" }) - vmray_results = {'results': vmray_results["results"] + p["results"] } + p["results"].append({"values": url1 + url2 + url3, "types": "link"}) + vmray_results = {'results': vmray_results["results"] + p["results"]} # Clean up (remove doubles) vmray_results = vmrayCleanup(vmray_results) @@ -121,7 +115,6 @@ def handler(q=False): return misperrors - def introspection(): modulesetup = {} try: @@ -175,42 +168,42 @@ def vmrayVtiPatterns(vti_patterns): elif pattern["category"] == "_network" and pattern["operation"] == "_connect": content = vmrayConnect(pattern) - elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page": + elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page": content = vmrayGeneric(pattern) - elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint": + elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint": content = vmrayGeneric(pattern, "mutex", 1) - elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution": + elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution": content = vmrayGeneric(pattern) - elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage": + elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage": content = vmrayGeneric(pattern) - elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file": + elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file": content = vmrayGeneric(pattern, "filename", 1) - elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file": + elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file": content = vmrayGeneric(pattern, "filename", 1) - elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory": + elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory": content = vmrayGeneric(pattern) - elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow": + elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow": content = vmrayGeneric(pattern) - elif only_network_info == False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files": + elif only_network_info is False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files": content = vmrayGeneric(pattern) - elif only_network_info == False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script": + elif only_network_info is False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script": content = vmrayGeneric(pattern, "regkey", 1) - elif only_network_info == False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges": + elif only_network_info is False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges": content = vmrayGeneric(pattern) if content: - r["results"].append( content["attributes"] ) - r["results"].append( content["text"] ) + r["results"].append(content["attributes"]) + r["results"].append(content["text"]) # Remove empty results - r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0] + r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0] for el in r["results"]: - if not el in y["results"]: - y["results"].append( el ) + if el not in y["results"]: + y["results"].append(el) return y else: return False @@ -221,22 +214,22 @@ def vmrayCleanup(x): y = {'results': []} for el in x["results"]: - if not el in y["results"]: - y["results"].append( el ) + if el not in y["results"]: + y["results"].append(el) return y def vmraySanitizeInput(s): ''' Sanitize some input so it gets properly imported in MISP''' if s: - s = s.replace('"','') - s = re.sub('\\\\', r'\\', s) + s = s.replace('"', '') + s = re.sub('\\\\', r'\\', s) return s else: return False -def vmrayGeneric(el, attr = "", attrpos = 1): +def vmrayGeneric(el, attr="", attrpos=1): ''' Convert a 'generic' VTI pattern to MISP data''' r = {"values": []} @@ -257,8 +250,7 @@ def vmrayGeneric(el, attr = "", attrpos = 1): f["values"].append(vmraySanitizeInput(content)) f["types"] = ["text"] - return { "text": f, - "attributes": r} + return {"text": f, "attributes": r} else: return False else: @@ -276,7 +268,7 @@ def vmrayConnect(el): content = el["technique_desc"] if content: target = content.split("\"") - port = (target[1].split(":"))[1] + # port = (target[1].split(":"))[1] ## FIXME: not used host = (target[1].split(":"))[0] if ipre.match(str(host)): r["values"].append(host) @@ -292,8 +284,7 @@ def vmrayConnect(el): f["values"].append(vmraySanitizeInput(content)) f["types"] = ["text"] - return { "text": f, - "attributes": r} + return {"text": f, "attributes": r} else: return False else: