MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Multiple clanges in the vmray modules. 

* Generic fix to load modules requiring a local library
* Fix python3 support
* PEP8 related cleanups
pull/70/head
Raphaël Vinot 2016-11-15 16:43:11 +01:00
parent ddcc1ae6e6
commit c676587461
9 changed files with 67 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
misp_modules/__init__.py
View File

@ -123,6 +123,8 @@ def load_modules(mod_dir):
if os.path.basename(root).startswith("."):
continue
for filename in fnmatch.filter(filenames, '*.py'):
if root.split('/')[-1].startswith('_'):
continue
if filename == '__init__.py':
continue
modulename = filename.split(".")[0]
@ -145,7 +147,7 @@ def load_package_modules():
mhandlers = {}
modules = []
for path, module in sys.modules.items():
r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
r = re.findall("misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
if r and len(r[0]) == 2:
moduletype, modulename = r[0]
mhandlers[modulename] = module

4
misp_modules/modules/expansion/__init__.py
View File

@ -1,2 +1,4 @@
__all__ = ['vmray_submit','asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
from . import _vmray
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'wiki']

0
misp_modules/modules/expansion/_vmray/__init__.py Normal file
View File

8
misp_modules/modules/import_mod/vmray_rest_api.py → misp_modules/modules/expansion/_vmray/vmray_rest_api.py
View File

@ -1,16 +1,12 @@
#!/usr/bin/python3
#!/usr/bin/env python3
"""Python client library for VMRay REST API"""
import base64
import datetime
import os.path
import requests
#import urlparse
import urllib.parse
from io import IOBase
from io import BytesIO
# disable nasty certification warning
# pylint: disable=no-member
try:
@ -87,7 +83,7 @@ class VMRayRESTAPI(object):
filename = value["filename"]
sample = value["data"]
file_params[key] = (filename, sample, "application/octet-stream")
elif isinstance(value, file) or hasattr(value, "read"):
elif hasattr(value, "read"):
filename = os.path.split(value.name)[1]
# For the following block refer to DEV-1820
try:

27
misp_modules/modules/expansion/vmray_submit.py
View File

@ -12,16 +12,12 @@ TODO:
'''
import json
import re
import base64
import sys
import os
base_dir = os.path.dirname(__file__) or '.'
sys.path.append(base_dir)
from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError
import io
from ._vmray.vmray_rest_api import VMRayRESTAPI
misperrors = {'error': 'Error'}
mispattributes = {'input': ['attachment'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']}
moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe',
@ -81,7 +77,7 @@ def handler(q=False):
if data and attachment:
args = {}
args["shareable"] = shareable
args["sample_file"] = {'data': io.BytesIO( data ) , 'filename': attachment }
args["sample_file"] = {'data': io.BytesIO(data), 'filename': attachment}
args["reanalyze"] = reanalyze
try:
@ -118,13 +114,13 @@ def vmrayProcess(vmraydata):
# Result received?
if submissions and jobs:
r = {'results': []}
r["results"].append( {"types": "md5", "values": submissions["submission_sample_md5"]} )
r["results"].append( {"types": "sha1", "values": submissions["submission_sample_sha1"]} )
r["results"].append( {"types": "sha256", "values": submissions["submission_sample_sha256"]} )
r["results"].append( {"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]} )
r["results"].append( {"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]} )
r["results"].append( {"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]} )
r["results"].append( {"types": "link", "values": submissions["submission_webif_url"]} )
r["results"].append({"types": "md5", "values": submissions["submission_sample_md5"]})
r["results"].append({"types": "sha1", "values": submissions["submission_sample_sha1"]})
r["results"].append({"types": "sha256", "values": submissions["submission_sample_sha256"]})
r["results"].append({"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]})
r["results"].append({"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]})
r["results"].append({"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]})
r["results"].append({"types": "link", "values": submissions["submission_webif_url"]})
# Include data from different jobs
if include_vmrayjobids:
@ -132,7 +128,7 @@ def vmrayProcess(vmraydata):
job_id = job["job_id"]
job_vm_name = job["job_vm_name"]
job_configuration_name = job["job_configuration_name"]
r["results"].append( {"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name) })
r["results"].append({"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name)})
return r
else:
misperrors['error'] = "No valid results returned."
@ -149,4 +145,3 @@ def vmraySubmit(api, args):
''' Submit the sample to VMRay'''
vmraydata = api.call("POST", "/rest/sample/submit", args)
return vmraydata

4
misp_modules/modules/import_mod/__init__.py
View File

@ -1 +1,3 @@
__all__ = ['vmray_import','testimport', 'ocr', 'stiximport']
from . import _vmray
__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport']

0
misp_modules/modules/import_mod/_vmray/__init__.py Normal file
View File

6
misp_modules/modules/expansion/vmray_rest_api.py → misp_modules/modules/import_mod/_vmray/vmray_rest_api.py
View File

@ -5,12 +5,8 @@ import base64
import datetime
import os.path
import requests
#import urlparse
import urllib.parse
from io import IOBase
from io import BytesIO
# disable nasty certification warning
# pylint: disable=no-member
try:
@ -87,7 +83,7 @@ class VMRayRESTAPI(object):
filename = value["filename"]
sample = value["data"]
file_params[key] = (filename, sample, "application/octet-stream")
elif isinstance(value, file) or hasattr(value, "read"):
elif hasattr(value, "read"):
filename = os.path.split(value.name)[1]
# For the following block refer to DEV-1820
try:

97
misp_modules/modules/import_mod/vmray_import.py
View File

@ -16,42 +16,36 @@ TODO:
import json
import re
import sys
import os
base_dir = os.path.dirname(__file__) or '.'
sys.path.append(base_dir)
from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError
from ._vmray.vmray_rest_api import VMRayRESTAPI
misperrors = {'error': 'Error'}
inputSource = []
moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe',
'description': 'Import VMRay (VTI) results',
'module-type': ['import']}
userConfig = {
'include_textdescr': {
'type': 'Boolean',
'message': 'Include textual description'
},
'include_analysisid': {
'type': 'Boolean',
'message': 'Include VMRay analysis_id text'
},
'only_network_info': {
'type': 'Boolean',
'message': 'Only include network (src-ip, hostname, domain, ...) information'
},
'sample_id': {
'type': 'Integer',
'errorMessage': 'Expected a sample ID',
'message': 'The VMRay sample_id'
}
};
userConfig = {'include_textdescr': {'type': 'Boolean',
'message': 'Include textual description'
},
'include_analysisid': {'type': 'Boolean',
'message': 'Include VMRay analysis_id text'
},
'only_network_info': {'type': 'Boolean',
'message': 'Only include network (src-ip, hostname, domain, ...) information'
},
'sample_id': {'type': 'Integer',
'errorMessage': 'Expected a sample ID',
'message': 'The VMRay sample_id'
}
}
moduleconfig = ['apikey', 'url']
include_textdescr = False
include_analysisid = False
only_network_info = False
def handler(q=False):
global include_textdescr
global include_analysisid
@ -104,8 +98,8 @@ def handler(q=False):
url1 = "https://cloud.vmray.com/user/analysis/view?from_sample_id=%u" % sample_id
url2 = "&id=%u" % analysis_id
url3 = "&sub=%2Freport%2Foverview.html"
p["results"].append({ "values": url1 + url2 + url3, "types": "link" })
vmray_results = {'results': vmray_results["results"] + p["results"] }
p["results"].append({"values": url1 + url2 + url3, "types": "link"})
vmray_results = {'results': vmray_results["results"] + p["results"]}
# Clean up (remove doubles)
vmray_results = vmrayCleanup(vmray_results)
@ -121,7 +115,6 @@ def handler(q=False):
return misperrors
def introspection():
modulesetup = {}
try:
@ -175,42 +168,42 @@ def vmrayVtiPatterns(vti_patterns):
elif pattern["category"] == "_network" and pattern["operation"] == "_connect":
content = vmrayConnect(pattern)
elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page":
elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint":
elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint":
content = vmrayGeneric(pattern, "mutex", 1)
elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage":
elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file":
elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file":
content = vmrayGeneric(pattern, "filename", 1)
elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file":
elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file":
content = vmrayGeneric(pattern, "filename", 1)
elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory":
elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow":
elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files":
elif only_network_info is False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script":
elif only_network_info is False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script":
content = vmrayGeneric(pattern, "regkey", 1)
elif only_network_info == False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges":
elif only_network_info is False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges":
content = vmrayGeneric(pattern)
if content:
r["results"].append( content["attributes"] )
r["results"].append( content["text"] )
r["results"].append(content["attributes"])
r["results"].append(content["text"])
# Remove empty results
r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0]
r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0]
for el in r["results"]:
if not el in y["results"]:
y["results"].append( el )
if el not in y["results"]:
y["results"].append(el)
return y
else:
return False
@ -221,22 +214,22 @@ def vmrayCleanup(x):
y = {'results': []}
for el in x["results"]:
if not el in y["results"]:
y["results"].append( el )
if el not in y["results"]:
y["results"].append(el)
return y
def vmraySanitizeInput(s):
''' Sanitize some input so it gets properly imported in MISP'''
if s:
s = s.replace('"','')
s = re.sub('\\\\', r'\\', s)
s = s.replace('"', '')
s = re.sub('\\\\', r'\\', s)
return s
else:
return False
def vmrayGeneric(el, attr = "", attrpos = 1):
def vmrayGeneric(el, attr="", attrpos=1):
''' Convert a 'generic' VTI pattern to MISP data'''
r = {"values": []}
@ -257,8 +250,7 @@ def vmrayGeneric(el, attr = "", attrpos = 1):
f["values"].append(vmraySanitizeInput(content))
f["types"] = ["text"]
return { "text": f,
"attributes": r}
return {"text": f, "attributes": r}
else:
return False
else:
@ -276,7 +268,7 @@ def vmrayConnect(el):
content = el["technique_desc"]
if content:
target = content.split("\"")
port = (target[1].split(":"))[1]
# port = (target[1].split(":"))[1] ## FIXME: not used
host = (target[1].split(":"))[0]
if ipre.match(str(host)):
r["values"].append(host)
@ -292,8 +284,7 @@ def vmrayConnect(el):
f["values"].append(vmraySanitizeInput(content))
f["types"] = ["text"]
return { "text": f,
"attributes": r}
return {"text": f, "attributes": r}
else:
return False
else: