MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Multiple clanges in the vmray modules.

Browse Source 
* Generic fix to load modules requiring a local library
* Fix python3 support
* PEP8 related cleanups
pull/70/head
Raphaël Vinot 7 years ago
parent ddcc1ae6e6
commit c676587461
9 changed files with 67 additions and 83 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      misp_modules/__init__.py
  2. 4
      misp_modules/modules/expansion/__init__.py
  3. 0
      misp_modules/modules/expansion/_vmray/__init__.py
  4. 8
      misp_modules/modules/expansion/_vmray/vmray_rest_api.py
  5. 27
      misp_modules/modules/expansion/vmray_submit.py
  6. 4
      misp_modules/modules/import_mod/__init__.py
  7. 0
      misp_modules/modules/import_mod/_vmray/__init__.py
  8. 6
      misp_modules/modules/import_mod/_vmray/vmray_rest_api.py
  9. 97
      misp_modules/modules/import_mod/vmray_import.py

4
misp_modules/__init__.py
Unescape View File

@ -123,6 +123,8 @@ def load_modules(mod_dir):
if os.path.basename(root).startswith("."):
continue
for filename in fnmatch.filter(filenames, '*.py'):
if root.split('/')[-1].startswith('_'):
continue
if filename == '__init__.py':
continue
modulename = filename.split(".")[0]
@ -145,7 +147,7 @@ def load_package_modules():
mhandlers = {}
modules = []
for path, module in sys.modules.items():
r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
r = re.findall("misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
if r and len(r[0]) == 2:
moduletype, modulename = r[0]
mhandlers[modulename] = module

4
misp_modules/modules/expansion/__init__.py
Unescape View File

@ -1,2 +1,4 @@
__all__ = ['vmray_submit','asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
from . import _vmray
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'wiki']

0
misp_modules/modules/expansion/_vmray/__init__.py
Unescape View File

8
misp_modules/modules/import_mod/vmray_rest_api.py → misp_modules/modules/expansion/_vmray/vmray_rest_api.py
Unescape View File

@ -1,16 +1,12 @@
#!/usr/bin/python3
#!/usr/bin/env python3
"""Python client library for VMRay REST API"""
import base64
import datetime
import os.path
import requests
#import urlparse
import urllib.parse
from io import IOBase
from io import BytesIO
# disable nasty certification warning
# pylint: disable=no-member
try:
@ -87,7 +83,7 @@ class VMRayRESTAPI(object):
filename = value["filename"]
sample = value["data"]
file_params[key] = (filename, sample, "application/octet-stream")
elif isinstance(value, file) or hasattr(value, "read"):
elif hasattr(value, "read"):
filename = os.path.split(value.name)[1]
# For the following block refer to DEV-1820
try:

27
misp_modules/modules/expansion/vmray_submit.py
Unescape View File

@ -12,16 +12,12 @@ TODO:
'''
import json
import re
import base64
import sys
import os
base_dir = os.path.dirname(__file__) or '.'
sys.path.append(base_dir)
from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError
import io
from ._vmray.vmray_rest_api import VMRayRESTAPI
misperrors = {'error': 'Error'}
mispattributes = {'input': ['attachment'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']}
moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe',
@ -81,7 +77,7 @@ def handler(q=False):
if data and attachment:
args = {}
args["shareable"] = shareable
args["sample_file"] = {'data': io.BytesIO( data ) , 'filename': attachment }
args["sample_file"] = {'data': io.BytesIO(data), 'filename': attachment}
args["reanalyze"] = reanalyze
try:
@ -118,13 +114,13 @@ def vmrayProcess(vmraydata):
# Result received?
if submissions and jobs:
r = {'results': []}
r["results"].append( {"types": "md5", "values": submissions["submission_sample_md5"]} )
r["results"].append( {"types": "sha1", "values": submissions["submission_sample_sha1"]} )
r["results"].append( {"types": "sha256", "values": submissions["submission_sample_sha256"]} )
r["results"].append( {"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]} )
r["results"].append( {"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]} )
r["results"].append( {"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]} )
r["results"].append( {"types": "link", "values": submissions["submission_webif_url"]} )
r["results"].append({"types": "md5", "values": submissions["submission_sample_md5"]})
r["results"].append({"types": "sha1", "values": submissions["submission_sample_sha1"]})
r["results"].append({"types": "sha256", "values": submissions["submission_sample_sha256"]})
r["results"].append({"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]})
r["results"].append({"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]})
r["results"].append({"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]})
r["results"].append({"types": "link", "values": submissions["submission_webif_url"]})
# Include data from different jobs
if include_vmrayjobids:
@ -132,7 +128,7 @@ def vmrayProcess(vmraydata):
job_id = job["job_id"]
job_vm_name = job["job_vm_name"]
job_configuration_name = job["job_configuration_name"]
r["results"].append( {"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name) })
r["results"].append({"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name)})
return r
else:
misperrors['error'] = "No valid results returned."
@ -149,4 +145,3 @@ def vmraySubmit(api, args):
''' Submit the sample to VMRay'''
vmraydata = api.call("POST", "/rest/sample/submit", args)
return vmraydata

4
misp_modules/modules/import_mod/__init__.py
Unescape View File

@ -1 +1,3 @@
__all__ = ['vmray_import','testimport', 'ocr', 'stiximport']
from . import _vmray
__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport']

0
misp_modules/modules/import_mod/_vmray/__init__.py
Unescape View File

6
misp_modules/modules/expansion/vmray_rest_api.py → misp_modules/modules/import_mod/_vmray/vmray_rest_api.py
Unescape View File

@ -5,12 +5,8 @@ import base64
import datetime
import os.path
import requests
#import urlparse
import urllib.parse
from io import IOBase
from io import BytesIO
# disable nasty certification warning
# pylint: disable=no-member
try:
@ -87,7 +83,7 @@ class VMRayRESTAPI(object):
filename = value["filename"]
sample = value["data"]
file_params[key] = (filename, sample, "application/octet-stream")
elif isinstance(value, file) or hasattr(value, "read"):
elif hasattr(value, "read"):
filename = os.path.split(value.name)[1]
# For the following block refer to DEV-1820
try:

97
misp_modules/modules/import_mod/vmray_import.py
Unescape View File

@ -16,42 +16,36 @@ TODO:
import json
import re
import sys
import os
base_dir = os.path.dirname(__file__) or '.'
sys.path.append(base_dir)
from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError
from ._vmray.vmray_rest_api import VMRayRESTAPI
misperrors = {'error': 'Error'}
inputSource = []
moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe',
'description': 'Import VMRay (VTI) results',
'module-type': ['import']}
userConfig = {
'include_textdescr': {
'type': 'Boolean',
'message': 'Include textual description'
},
'include_analysisid': {
'type': 'Boolean',
'message': 'Include VMRay analysis_id text'
},
'only_network_info': {
'type': 'Boolean',
'message': 'Only include network (src-ip, hostname, domain, ...) information'
},
'sample_id': {
'type': 'Integer',
'errorMessage': 'Expected a sample ID',
'message': 'The VMRay sample_id'
}
};
userConfig = {'include_textdescr': {'type': 'Boolean',
'message': 'Include textual description'
},
'include_analysisid': {'type': 'Boolean',
'message': 'Include VMRay analysis_id text'
},
'only_network_info': {'type': 'Boolean',
'message': 'Only include network (src-ip, hostname, domain, ...) information'
},
'sample_id': {'type': 'Integer',
'errorMessage': 'Expected a sample ID',
'message': 'The VMRay sample_id'
}
}
moduleconfig = ['apikey', 'url']
include_textdescr = False
include_analysisid = False
only_network_info = False
def handler(q=False):
global include_textdescr
global include_analysisid
@ -104,8 +98,8 @@ def handler(q=False):
url1 = "https://cloud.vmray.com/user/analysis/view?from_sample_id=%u" % sample_id
url2 = "&id=%u" % analysis_id
url3 = "&sub=%2Freport%2Foverview.html"
p["results"].append({ "values": url1 + url2 + url3, "types": "link" })
vmray_results = {'results': vmray_results["results"] + p["results"] }
p["results"].append({"values": url1 + url2 + url3, "types": "link"})
vmray_results = {'results': vmray_results["results"] + p["results"]}
# Clean up (remove doubles)
vmray_results = vmrayCleanup(vmray_results)
@ -121,7 +115,6 @@ def handler(q=False):
return misperrors
def introspection():
modulesetup = {}
try:
@ -175,42 +168,42 @@ def vmrayVtiPatterns(vti_patterns):
elif pattern["category"] == "_network" and pattern["operation"] == "_connect":
content = vmrayConnect(pattern)
elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page":
elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint":
elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint":
content = vmrayGeneric(pattern, "mutex", 1)
elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage":
elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file":
elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file":
content = vmrayGeneric(pattern, "filename", 1)
elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file":
elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file":
content = vmrayGeneric(pattern, "filename", 1)
elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory":
elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow":
elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files":
elif only_network_info is False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script":
elif only_network_info is False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script":
content = vmrayGeneric(pattern, "regkey", 1)
elif only_network_info == False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges":
elif only_network_info is False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges":
content = vmrayGeneric(pattern)
if content:
r["results"].append( content["attributes"] )
r["results"].append( content["text"] )
r["results"].append(content["attributes"])
r["results"].append(content["text"])
# Remove empty results
r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0]
r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0]
for el in r["results"]:
if not el in y["results"]:
y["results"].append( el )
if el not in y["results"]:
y["results"].append(el)
return y
else:
return False
@ -221,22 +214,22 @@ def vmrayCleanup(x):
y = {'results': []}
for el in x["results"]:
if not el in y["results"]:
y["results"].append( el )
if el not in y["results"]:
y["results"].append(el)
return y
def vmraySanitizeInput(s):
''' Sanitize some input so it gets properly imported in MISP'''
if s:
s = s.replace('"','')
s = re.sub('\\\\', r'\\', s)
s = s.replace('"', '')
s = re.sub('\\\\', r'\\', s)
return s
else:
return False
def vmrayGeneric(el, attr = "", attrpos = 1):
def vmrayGeneric(el, attr="", attrpos=1):
''' Convert a 'generic' VTI pattern to MISP data'''
r = {"values": []}
@ -257,8 +250,7 @@ def vmrayGeneric(el, attr = "", attrpos = 1):
f["values"].append(vmraySanitizeInput(content))
f["types"] = ["text"]
return { "text": f,
"attributes": r}
return {"text": f, "attributes": r}
else:
return False
else:
@ -276,7 +268,7 @@ def vmrayConnect(el):
content = el["technique_desc"]
if content:
target = content.split("\"")
port = (target[1].split(":"))[1]
# port = (target[1].split(":"))[1] ## FIXME: not used
host = (target[1].split(":"))[0]
if ipre.match(str(host)):
r["values"].append(host)
@ -292,8 +284,7 @@ def vmrayConnect(el):
f["values"].append(vmraySanitizeInput(content))
f["types"] = ["text"]
return { "text": f,
"attributes": r}
return {"text": f, "attributes": r}
else:
return False
else:

Write Preview
Loading…
Cancel
Save