MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Sync

pull/69/head
Koen Van Impe 2016-11-18 18:09:58 +01:00
parent 7c05a8acef
commit 9d48571566
11 changed files with 102 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
misp_modules/__init__.py
View File

@ -123,6 +123,8 @@ def load_modules(mod_dir):
if os.path.basename(root).startswith("."):
continue
for filename in fnmatch.filter(filenames, '*.py'):
if root.split('/')[-1].startswith('_'):
continue
if filename == '__init__.py':
continue
modulename = filename.split(".")[0]
@ -145,7 +147,7 @@ def load_package_modules():
mhandlers = {}
modules = []
for path, module in sys.modules.items():
r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
r = re.findall("misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
if r and len(r[0]) == 2:
moduletype, modulename = r[0]
mhandlers[modulename] = module

4
misp_modules/modules/expansion/__init__.py
View File

@ -1,2 +1,4 @@
__all__ = ['vmray_submit','asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
from . import _vmray
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns',
'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'wiki']

0
misp_modules/modules/expansion/_vmray/__init__.py Normal file
View File

8
misp_modules/modules/import_mod/vmray_rest_api.py → misp_modules/modules/expansion/_vmray/vmray_rest_api.py
View File

@ -1,16 +1,12 @@
#!/usr/bin/python3
#!/usr/bin/env python3
"""Python client library for VMRay REST API"""
import base64
import datetime
import os.path
import requests
#import urlparse
import urllib.parse
from io import IOBase
from io import BytesIO
# disable nasty certification warning
# pylint: disable=no-member
try:
@ -87,7 +83,7 @@ class VMRayRESTAPI(object):
filename = value["filename"]
sample = value["data"]
file_params[key] = (filename, sample, "application/octet-stream")
elif isinstance(value, file) or hasattr(value, "read"):
elif hasattr(value, "read"):
filename = os.path.split(value.name)[1]
# For the following block refer to DEV-1820
try:

29
misp_modules/modules/expansion/vmray_submit.py
View File

@ -12,8 +12,8 @@ TODO:
'''
import json
import re
import base64
<<<<<<< HEAD
import io
import sys
import os
@ -24,6 +24,12 @@ base_dir = os.path.dirname(__file__) or '.'
sys.path.append(base_dir)
from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError
=======
import io
>>>>>>> upstream/master
from ._vmray.vmray_rest_api import VMRayRESTAPI
misperrors = {'error': 'Error'}
mispattributes = {'input': ['attachment', 'malware-sample'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']}
@ -98,7 +104,11 @@ def handler(q=False):
if data and sample_filename:
args = {}
args["shareable"] = shareable
<<<<<<< HEAD
args["sample_file"] = {'data': io.BytesIO( data ) , 'filename': sample_filename }
=======
args["sample_file"] = {'data': io.BytesIO(data), 'filename': attachment}
>>>>>>> upstream/master
args["reanalyze"] = reanalyze
try:
@ -135,13 +145,13 @@ def vmrayProcess(vmraydata):
# Result received?
if submissions and jobs:
r = {'results': []}
r["results"].append( {"types": "md5", "values": submissions["submission_sample_md5"]} )
r["results"].append( {"types": "sha1", "values": submissions["submission_sample_sha1"]} )
r["results"].append( {"types": "sha256", "values": submissions["submission_sample_sha256"]} )
r["results"].append( {"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]} )
r["results"].append( {"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]} )
r["results"].append( {"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]} )
r["results"].append( {"types": "link", "values": submissions["submission_webif_url"]} )
r["results"].append({"types": "md5", "values": submissions["submission_sample_md5"]})
r["results"].append({"types": "sha1", "values": submissions["submission_sample_sha1"]})
r["results"].append({"types": "sha256", "values": submissions["submission_sample_sha256"]})
r["results"].append({"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]})
r["results"].append({"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]})
r["results"].append({"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]})
r["results"].append({"types": "link", "values": submissions["submission_webif_url"]})
# Include data from different jobs
if include_vmrayjobids:
@ -149,7 +159,7 @@ def vmrayProcess(vmraydata):
job_id = job["job_id"]
job_vm_name = job["job_vm_name"]
job_configuration_name = job["job_configuration_name"]
r["results"].append( {"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name) })
r["results"].append({"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name)})
return r
else:
misperrors['error'] = "No valid results returned."
@ -166,4 +176,3 @@ def vmraySubmit(api, args):
''' Submit the sample to VMRay'''
vmraydata = api.call("POST", "/rest/sample/submit", args)
return vmraydata

4
misp_modules/modules/import_mod/__init__.py
View File

@ -1 +1,3 @@
__all__ = ['vmray_import','testimport', 'ocr', 'stiximport']
from . import _vmray
__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport']

0
misp_modules/modules/import_mod/_vmray/__init__.py Normal file
View File

6
misp_modules/modules/expansion/vmray_rest_api.py → misp_modules/modules/import_mod/_vmray/vmray_rest_api.py
View File

@ -5,12 +5,8 @@ import base64
import datetime
import os.path
import requests
#import urlparse
import urllib.parse
from io import IOBase
from io import BytesIO
# disable nasty certification warning
# pylint: disable=no-member
try:
@ -87,7 +83,7 @@ class VMRayRESTAPI(object):
filename = value["filename"]
sample = value["data"]
file_params[key] = (filename, sample, "application/octet-stream")
elif isinstance(value, file) or hasattr(value, "read"):
elif hasattr(value, "read"):
filename = os.path.split(value.name)[1]
# For the following block refer to DEV-1820
try:

9
misp_modules/modules/import_mod/stiximport.py
View File

@ -28,7 +28,7 @@ def handler(q=False):
q = json.loads(q)
# It's b64 encoded, so decode that stuff
package = str(base64.b64decode(q.get("data", None)), 'utf-8')
package = base64.b64decode(q.get("data")).decode('utf-8')
# If something really weird happened
if not package:
@ -168,6 +168,9 @@ def buildObservable(o):
# May as well be useless
return r
if not o.get('object'):
return r
props = o["object"]["properties"]
# If it has an address_value field, it's gonna be an address
@ -195,7 +198,7 @@ def buildObservable(o):
for hsh in props["hashes"]:
r["values"].append(hsh["simple_hash_value"]["value"])
r["types"] = identifyHash(hsh["simple_hash_value"]["value"])
elif "xsi:type" in props:
# Cybox. Ew.
try:
@ -208,7 +211,7 @@ def buildObservable(o):
else:
print("Ignoring {}".format(type_))
except:
pass
pass
return r

103
misp_modules/modules/import_mod/vmray_import.py
View File

@ -16,42 +16,36 @@ TODO:
import json
import re
import sys
import os
base_dir = os.path.dirname(__file__) or '.'
sys.path.append(base_dir)
from vmray_rest_api import VMRayRESTAPI, VMRayRESTAPIError
from ._vmray.vmray_rest_api import VMRayRESTAPI
misperrors = {'error': 'Error'}
inputSource = []
moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe',
'description': 'Import VMRay (VTI) results',
'module-type': ['import']}
userConfig = {
'include_textdescr': {
'type': 'Boolean',
'message': 'Include textual description'
},
'include_analysisid': {
'type': 'Boolean',
'message': 'Include VMRay analysis_id text'
},
'only_network_info': {
'type': 'Boolean',
'message': 'Only include network (src-ip, hostname, domain, ...) information'
},
'sample_id': {
'type': 'Integer',
'errorMessage': 'Expected a sample ID',
'message': 'The VMRay sample_id'
}
};
userConfig = {'include_textdescr': {'type': 'Boolean',
'message': 'Include textual description'
},
'include_analysisid': {'type': 'Boolean',
'message': 'Include VMRay analysis_id text'
},
'only_network_info': {'type': 'Boolean',
'message': 'Only include network (src-ip, hostname, domain, ...) information'
},
'sample_id': {'type': 'Integer',
'errorMessage': 'Expected a sample ID',
'message': 'The VMRay sample_id'
}
}
moduleconfig = ['apikey', 'url']
include_textdescr = False
include_analysisid = False
only_network_info = False
def handler(q=False):
global include_textdescr
global include_analysisid
@ -100,9 +94,19 @@ def handler(q=False):
if analysis_data:
p = vmrayVtiPatterns(analysis_data["vti_patterns"])
<<<<<<< HEAD
if p and len(p["results"]) > 0:
vti_patterns_found = True
vmray_results = {'results': vmray_results["results"] + p["results"] }
=======
if p:
if include_analysisid:
url1 = "https://cloud.vmray.com/user/analysis/view?from_sample_id=%u" % sample_id
url2 = "&id=%u" % analysis_id
url3 = "&sub=%2Freport%2Foverview.html"
p["results"].append({"values": url1 + url2 + url3, "types": "link"})
vmray_results = {'results': vmray_results["results"] + p["results"]}
>>>>>>> upstream/master
if include_analysisid:
a_id = {'results': []}
@ -130,7 +134,6 @@ def handler(q=False):
return misperrors
def introspection():
modulesetup = {}
try:
@ -184,44 +187,44 @@ def vmrayVtiPatterns(vti_patterns):
elif pattern["category"] == "_network" and pattern["operation"] == "_connect":
content = vmrayConnect(pattern)
elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page":
elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint":
elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint":
content = vmrayGeneric(pattern, "mutex", 1)
elif only_network_info == False and pattern["category"] == "_process" and pattern["operation"] == "_crashed_process":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage":
elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file":
elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file":
content = vmrayGeneric(pattern, "filename", 1)
elif only_network_info == False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file":
elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file":
content = vmrayGeneric(pattern, "filename", 1)
elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory":
elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow":
elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files":
elif only_network_info is False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files":
content = vmrayGeneric(pattern)
elif only_network_info == False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script":
elif only_network_info is False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script":
content = vmrayGeneric(pattern, "regkey", 1)
elif only_network_info == False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges":
elif only_network_info is False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges":
content = vmrayGeneric(pattern)
if content:
r["results"].append( content["attributes"] )
r["results"].append( content["text"] )
r["results"].append(content["attributes"])
r["results"].append(content["text"])
# Remove empty results
r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0]
r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0]
for el in r["results"]:
if not el in y["results"]:
y["results"].append( el )
if el not in y["results"]:
y["results"].append(el)
return y
else:
return False
@ -232,22 +235,22 @@ def vmrayCleanup(x):
y = {'results': []}
for el in x["results"]:
if not el in y["results"]:
y["results"].append( el )
if el not in y["results"]:
y["results"].append(el)
return y
def vmraySanitizeInput(s):
''' Sanitize some input so it gets properly imported in MISP'''
if s:
s = s.replace('"','')
s = re.sub('\\\\', r'\\', s)
s = s.replace('"', '')
s = re.sub('\\\\', r'\\', s)
return s
else:
return False
def vmrayGeneric(el, attr = "", attrpos = 1):
def vmrayGeneric(el, attr="", attrpos=1):
''' Convert a 'generic' VTI pattern to MISP data'''
r = {"values": []}
@ -270,8 +273,7 @@ def vmrayGeneric(el, attr = "", attrpos = 1):
f["values"].append(vmraySanitizeInput(content))
f["types"] = ["text"]
return { "text": f,
"attributes": r}
return {"text": f, "attributes": r}
else:
return False
else:
@ -289,7 +291,7 @@ def vmrayConnect(el):
content = el["technique_desc"]
if content:
target = content.split("\"")
port = (target[1].split(":"))[1]
# port = (target[1].split(":"))[1] ## FIXME: not used
host = (target[1].split(":"))[0]
if ipre.match(str(host)):
r["values"].append(host)
@ -305,8 +307,7 @@ def vmrayConnect(el):
f["values"].append(vmraySanitizeInput(content))
f["types"] = ["text"]
return { "text": f,
"attributes": r}
return {"text": f, "attributes": r}
else:
return False
else:

24
tests/test.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import unittest
@ -7,6 +7,7 @@ import base64
import json
import os
class TestModules(unittest.TestCase):
def setUp(self):
@ -32,13 +33,14 @@ class TestModules(unittest.TestCase):
print(response.json())
def test_stix(self):
with open("tests/stix.xml", "r") as f:
data = json.dumps({"module":"stiximport",
"data":str(base64.b64encode(bytes(f.read(), 'utf-8')), 'utf-8'),
"config": {"max_size": "15000"},
})
with open("tests/stix.xml", "rb") as f:
content = base64.b64encode(f.read())
data = json.dumps({"module": "stiximport",
"data": content.decode('utf-8'),
"config": {"max_size": "15000"},
})
response = requests.post(self.url + "query", data=data)
print(response.json())
print('STIX', response.json())
def test_virustotal(self):
# This can't actually be tested without disclosing a private
@ -46,11 +48,11 @@ class TestModules(unittest.TestCase):
# and pass if it can't find one
if not os.path.exists("tests/bodyvirustotal.json"):
return
return
with open("tests/bodyvirustotal.json", "r") as f:
response = requests.post(self.url + "query", data=f.read()).json()
response = requests.post(self.url + "query", data=f.read()).json()
assert(response)
if __name__ == '__main__':
unittest.main()
unittest.main()