MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Add OpenIOC import module

pull/114/head
Raphaël Vinot 2017-02-27 13:32:31 +01:00
parent 8bd9b46713
commit c508e60f65
5 changed files with 200 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
REQUIREMENTS
View File

@ -19,3 +19,4 @@ pytesseract
SPARQLWrapper SPARQLWrapper
domaintools_api domaintools_api
pygeoip pygeoip
bs4

3
misp_modules/modules/import_mod/__init__.py
View File

@ -1,3 +1,4 @@
from . import _vmray from . import _vmray
__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', 'email_import', 'mispjson'] __all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport',
'email_import', 'mispjson', 'openiocimport']

58
misp_modules/modules/import_mod/openiocimport.py Executable file
View File

@ -0,0 +1,58 @@
import json
import base64
from pymisp.tools import openioc
misperrors = {'error': 'Error'}
userConfig = {}
inputSource = ['file']
moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot',
'description': 'Import OpenIOC package',
'module-type': ['import']}
moduleconfig = []
def handler(q=False):
# Just in case we have no data
if q is False:
return False
# The return value
r = {'results': []}
# Load up that JSON
q = json.loads(q)
# It's b64 encoded, so decode that stuff
package = base64.b64decode(q.get("data")).decode('utf-8')
# If something really weird happened
if not package:
return json.dumps({"success": 0})
pkg = openioc.load_openioc(package)
for attrib in pkg.attributes:
r["results"].append({"values": [attrib.value], "types": [attrib.type], "categories": [attrib.category]})
return r
def introspection():
modulesetup = {}
try:
userConfig
modulesetup['userConfig'] = userConfig
except NameError:
pass
try:
inputSource
modulesetup['inputSource'] = inputSource
except NameError:
pass
return modulesetup
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

91
tests/openioc.xml Normal file
View File

@ -0,0 +1,91 @@
<?xml version="1.0" encoding="us-ascii"?>
<ioc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" id="ea3cab0c-72ad-40cc-abbf-90846fa4afec" last-modified="2011-11-04T19:35:05" xmlns="http://schemas.mandiant.com/2010/ioc">
<short_description>STUXNET VIRUS (METHODOLOGY)</short_description>
<description>Generic indicator for the stuxnet virus. When loaded, stuxnet spawns lsass.exe in a suspended state. The malware then maps in its own executable section and fixes up the CONTEXT to point to the newly mapped in section. This is a common task performed by malware and allows the malware to execute under the pretense of a known and trusted process.</description>
<keywords>methodology</keywords>
<authored_by>Mandiant</authored_by>
<authored_date>0001-01-01T00:00:00</authored_date>
<links />
<definition>
<Indicator operator="OR" id="73bc8d65-826b-48d2-b4a8-48918e29e323">
<IndicatorItem id="b9ef2559-cc59-4463-81d9-52800545e16e" condition="contains">
<Context document="FileItem" search="FileItem/PEInfo/Sections/Section/Name" type="mir" />
<Content type="string">.stub</Content>
</IndicatorItem>
<IndicatorItem id="156bc4b6-a2a1-4735-bfe8-6c8d1f7eae38" condition="contains">
<Context document="FileItem" search="FileItem/FileName" type="mir" />
<Content type="string">mdmcpq3.PNF</Content>
</IndicatorItem>
<IndicatorItem id="e57d9a5b-5e6a-41ec-87c8-ee67f3ed2e20" condition="contains">
<Context document="FileItem" search="FileItem/FileName" type="mir" />
<Content type="string">mdmeric3.PNF</Content>
</IndicatorItem>
<IndicatorItem id="63d7bee6-b575-4d56-8d43-1c5eac57658f" condition="contains">
<Context document="FileItem" search="FileItem/FileName" type="mir" />
<Content type="string">oem6C.PNF</Content>
</IndicatorItem>
<IndicatorItem id="e6bff12a-e23d-45ea-94bd-8289f806bea7" condition="contains">
<Context document="FileItem" search="FileItem/FileName" type="mir" />
<Content type="string">oem7A.PNF</Content>
</IndicatorItem>
<Indicator operator="AND" id="422ae9bf-a1ae-41f2-8e54-5b4c6f3e1598">
<IndicatorItem id="e93f1610-daaf-4311-bcf3-3aecef8271c0" condition="contains">
<Context document="DriverItem" search="DriverItem/DeviceItem/AttachedToDriverName" type="mir" />
<Content type="string">fs_rec.sys</Content>
</IndicatorItem>
<IndicatorItem id="72476f35-8dea-4bae-8239-7c22d05d664f" condition="contains">
<Context document="DriverItem" search="DriverItem/DeviceItem/AttachedToDriverName" type="mir" />
<Content type="string">mrxsmb.sys</Content>
</IndicatorItem>
<IndicatorItem id="f98ea5aa-9e23-4f18-b871-b3cf5ba153fe" condition="contains">
<Context document="DriverItem" search="DriverItem/DeviceItem/AttachedToDriverName" type="mir" />
<Content type="string">sr.sys</Content>
</IndicatorItem>
<IndicatorItem id="32f61140-0f58-43bc-8cdd-a25db75ca6c4" condition="contains">
<Context document="DriverItem" search="DriverItem/DeviceItem/AttachedToDriverName" type="mir" />
<Content type="string">fastfat.sys</Content>
</IndicatorItem>
</Indicator>
<Indicator operator="AND" id="eb585bf5-18d8-4837-baf0-80ac74104ca3">
<IndicatorItem id="8d85b559-4d18-4e15-b0c9-da5a9b32f53c" condition="contains">
<Context document="FileItem" search="FileItem/FileName" type="mir" />
<Content type="string">mrxcls.sys</Content>
</IndicatorItem>
<IndicatorItem id="8a3e425d-fa87-4a31-b20d-8f8630d77933" condition="contains">
<Context document="FileItem" search="FileItem/PEInfo/DigitalSignature/CertificateSubject" type="mir" />
<Content type="string">Realtek Semiconductor Corp</Content>
</IndicatorItem>
</Indicator>
<Indicator operator="AND" id="bc8d06dd-f879-4609-bb1c-eccded0222ce">
<IndicatorItem id="89f194d3-3ee6-4218-93f8-055ea92a9f00" condition="contains">
<Context document="FileItem" search="FileItem/FileName" type="mir" />
<Content type="string">mrxnet.sys</Content>
</IndicatorItem>
<IndicatorItem id="c2dae8bf-81b1-49fb-8654-396830d75ade" condition="contains">
<Context document="FileItem" search="FileItem/PEInfo/DigitalSignature/CertificateSubject" type="mir" />
<Content type="string">Realtek Semiconductor Corp</Content>
</IndicatorItem>
</Indicator>
<Indicator operator="AND" id="00538c36-88fe-42ea-a70f-136a2fb53834">
<IndicatorItem id="a779b811-345f-4164-897e-0752837d0c1e" condition="contains">
<Context document="RegistryItem" search="RegistryItem/Path" type="mir" />
<Content type="string">HKEY_LOCAL_MACHINE\SYSTEM\ControlSet001\Services\MRxCls\ImagePath</Content>
</IndicatorItem>
<IndicatorItem id="ee981f06-b713-40aa-ac98-c6f4fd82b78d" condition="contains">
<Context document="RegistryItem" search="RegistryItem/Text" type="mir" />
<Content type="string">mrxcls.sys</Content>
</IndicatorItem>
</Indicator>
<Indicator operator="AND" id="d8d9b32c-f648-4552-9805-93c05ed48219">
<IndicatorItem id="c08044e7-e88c-433c-b463-763bdddeff82" condition="contains">
<Context document="RegistryItem" search="RegistryItem/Path" type="mir" />
<Content type="string">HKEY_LOCAL_MACHINE\SYSTEM\ControlSet001\Services\MRxNet\ImagePath</Content>
</IndicatorItem>
<IndicatorItem id="38dfb382-ebbe-4685-bbb7-60675b91bd15" condition="contains">
<Context document="RegistryItem" search="RegistryItem/Text" type="mir" />
<Content type="string">mrxnet.sys</Content>
</IndicatorItem>
</Indicator>
</Indicator>
</definition>
</ioc>

85
tests/test.py
View File

@ -41,6 +41,20 @@ class TestModules(unittest.TestCase):
print(response.json()) print(response.json())
response.connection.close() response.connection.close()
def test_openioc(self):
with open("tests/openioc.xml", "rb") as f:
content = base64.b64encode(f.read())
data = json.dumps({"module": "openiocimport",
"data": content.decode(),
})
response = requests.post(self.url + "query", data=data).json()
print(response)
print("OpenIOC :: {}".format(response))
values = [x["values"][0] for x in response["results"]]
assert("mrxcls.sys" in values)
assert("mdmcpq3.PNF" in values)
def test_stix(self): def test_stix(self):
with open("tests/stix.xml", "rb") as f: with open("tests/stix.xml", "rb") as f:
content = base64.b64encode(f.read()) content = base64.b64encode(f.read())
@ -57,7 +71,7 @@ class TestModules(unittest.TestCase):
assert("eu-society.com" in values) assert("eu-society.com" in values)
def test_email_headers(self): def test_email_headers(self):
query = {"module":"email_import"} query = {"module": "email_import"}
query["config"] = {"unzip_attachments": None, query["config"] = {"unzip_attachments": None,
"guess_zip_attachment_passwords": None, "guess_zip_attachment_passwords": None,
"extract_urls": None} "extract_urls": None}
@ -105,7 +119,7 @@ class TestModules(unittest.TestCase):
self.assertIn("<CI7DgL-A6dm92s7gf4-88g@E_0x238G4K2H08H9SDwsw8b6LwuA@mail.example.com>", values) self.assertIn("<CI7DgL-A6dm92s7gf4-88g@E_0x238G4K2H08H9SDwsw8b6LwuA@mail.example.com>", values)
def test_email_attachment_basic(self): def test_email_attachment_basic(self):
query = {"module":"email_import"} query = {"module": "email_import"}
query["config"] = {"unzip_attachments": None, query["config"] = {"unzip_attachments": None,
"guess_zip_attachment_passwords": None, "guess_zip_attachment_passwords": None,
"extract_urls": None} "extract_urls": None}
@ -128,9 +142,8 @@ class TestModules(unittest.TestCase):
attch_data = base64.b64decode(i["data"]) attch_data = base64.b64decode(i["data"])
self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-')
def test_email_attachment_unpack(self): def test_email_attachment_unpack(self):
query = {"module":"email_import"} query = {"module": "email_import"}
query["config"] = {"unzip_attachments": "true", query["config"] = {"unzip_attachments": "true",
"guess_zip_attachment_passwords": None, "guess_zip_attachment_passwords": None,
"extract_urls": None} "extract_urls": None}
@ -162,7 +175,7 @@ class TestModules(unittest.TestCase):
def test_email_dont_unpack_compressed_doc_attachments(self): def test_email_dont_unpack_compressed_doc_attachments(self):
"""Ensures that compressed """Ensures that compressed
""" """
query = {"module":"email_import"} query = {"module": "email_import"}
query["config"] = {"unzip_attachments": "true", query["config"] = {"unzip_attachments": "true",
"guess_zip_attachment_passwords": None, "guess_zip_attachment_passwords": None,
"extract_urls": None} "extract_urls": None}
@ -192,9 +205,8 @@ class TestModules(unittest.TestCase):
self.assertEqual(filesum.hexdigest(), self.assertEqual(filesum.hexdigest(),
'098da5381a90d4a51e6b844c18a0fecf2e364813c2f8b317cfdc51c21f2506a5') '098da5381a90d4a51e6b844c18a0fecf2e364813c2f8b317cfdc51c21f2506a5')
def test_email_attachment_unpack_with_password(self): def test_email_attachment_unpack_with_password(self):
query = {"module":"email_import"} query = {"module": "email_import"}
query["config"] = {"unzip_attachments": "true", query["config"] = {"unzip_attachments": "true",
"guess_zip_attachment_passwords": 'true', "guess_zip_attachment_passwords": 'true',
"extract_urls": None} "extract_urls": None}
@ -221,9 +233,8 @@ class TestModules(unittest.TestCase):
self.assertEqual(attch_data, self.assertEqual(attch_data,
b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-')
def test_email_attachment_password_in_body(self): def test_email_attachment_password_in_body(self):
query = {"module":"email_import"} query = {"module": "email_import"}
query["config"] = {"unzip_attachments": "true", query["config"] = {"unzip_attachments": "true",
"guess_zip_attachment_passwords": 'true', "guess_zip_attachment_passwords": 'true',
"extract_urls": None} "extract_urls": None}
@ -246,7 +257,7 @@ class TestModules(unittest.TestCase):
'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-')
def test_email_attachment_password_in_body_quotes(self): def test_email_attachment_password_in_body_quotes(self):
query = {"module":"email_import"} query = {"module": "email_import"}
query["config"] = {"unzip_attachments": "true", query["config"] = {"unzip_attachments": "true",
"guess_zip_attachment_passwords": 'true', "guess_zip_attachment_passwords": 'true',
"extract_urls": None} "extract_urls": None}
@ -274,7 +285,7 @@ class TestModules(unittest.TestCase):
'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-')
def test_email_attachment_password_in_html_body(self): def test_email_attachment_password_in_html_body(self):
query = {"module":"email_import"} query = {"module": "email_import"}
query["config"] = {"unzip_attachments": "true", query["config"] = {"unzip_attachments": "true",
"guess_zip_attachment_passwords": 'true', "guess_zip_attachment_passwords": 'true',
"extract_urls": None} "extract_urls": None}
@ -304,7 +315,7 @@ class TestModules(unittest.TestCase):
query['data'] = decode_email(message) query['data'] = decode_email(message)
data = json.dumps(query) data = json.dumps(query)
response = requests.post(self.url + "query", data=data) response = requests.post(self.url + "query", data=data)
#print(response.json()) # print(response.json())
values = [x["values"] for x in response.json()["results"]] values = [x["values"] for x in response.json()["results"]]
self.assertIn('EICAR.com', values) self.assertIn('EICAR.com', values)
for i in response.json()['results']: for i in response.json()['results']:
@ -315,7 +326,7 @@ class TestModules(unittest.TestCase):
'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-')
def test_email_attachment_password_in_subject(self): def test_email_attachment_password_in_subject(self):
query = {"module":"email_import"} query = {"module": "email_import"}
query["config"] = {"unzip_attachments": "true", query["config"] = {"unzip_attachments": "true",
"guess_zip_attachment_passwords": 'true', "guess_zip_attachment_passwords": 'true',
"extract_urls": None} "extract_urls": None}
@ -344,9 +355,8 @@ class TestModules(unittest.TestCase):
self.assertEqual(attch_data, self.assertEqual(attch_data,
'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-')
def test_email_extract_html_body_urls(self): def test_email_extract_html_body_urls(self):
query = {"module":"email_import"} query = {"module": "email_import"}
query["config"] = {"unzip_attachments": None, query["config"] = {"unzip_attachments": None,
"guess_zip_attachment_passwords": None, "guess_zip_attachment_passwords": None,
"extract_urls": "true"} "extract_urls": "true"}
@ -374,12 +384,12 @@ without modifying core components. The API is available via a simple REST API wh
query['data'] = decode_email(message) query['data'] = decode_email(message)
data = json.dumps(query) data = json.dumps(query)
response = requests.post(self.url + "query", data=data) response = requests.post(self.url + "query", data=data)
#print(response.json()) # print(response.json())
values = [x["values"] for x in response.json()["results"]] values = [x["values"] for x in response.json()["results"]]
self.assertIn("https://github.com/MISP/MISP", values) self.assertIn("https://github.com/MISP/MISP", values)
self.assertIn("https://www.circl.lu/assets/files/misp-training/3.1-MISP-modules.pdf", values) self.assertIn("https://www.circl.lu/assets/files/misp-training/3.1-MISP-modules.pdf", values)
#def test_domaintools(self): # def test_domaintools(self):
# query = {'config': {'username': 'test_user', 'api_key': 'test_key'}, 'module': 'domaintools', 'domain': 'domaintools.com'} # query = {'config': {'username': 'test_user', 'api_key': 'test_key'}, 'module': 'domaintools', 'domain': 'domaintools.com'}
# try: # try:
# response = requests.post(self.url + "query", data=json.dumps(query)).json() # response = requests.post(self.url + "query", data=json.dumps(query)).json()
@ -388,33 +398,34 @@ without modifying core components. The API is available via a simple REST API wh
# response = requests.post(self.url + "query", data=json.dumps(query)).json() # response = requests.post(self.url + "query", data=json.dumps(query)).json()
# print(response) # print(response)
def decode_email(message): def decode_email(message):
message64 = base64.b64encode(message.as_bytes()).decode() message64 = base64.b64encode(message.as_bytes()).decode()
return message64 return message64
def get_base_email(): def get_base_email():
headers = {"Received":"via dmail-2008.19 for +INBOX; Tue, 3 Feb 2009 19:29:12 -0600 (CST)", headers = {"Received": "via dmail-2008.19 for +INBOX; Tue, 3 Feb 2009 19:29:12 -0600 (CST)",
"Received":"from abc.luxsci.com ([10.10.10.10]) by xyz.luxsci.com (8.13.7/8.13.7) with ESMTP id n141TCa7022588 for <test@domain.com>; Tue, 3 Feb 2009 19:29:12 -0600", "Received": "from abc.luxsci.com ([10.10.10.10]) by xyz.luxsci.com (8.13.7/8.13.7) with ESMTP id n141TCa7022588 for <test@domain.com>; Tue, 3 Feb 2009 19:29:12 -0600",
"Received":"from [192.168.0.3] (verizon.net [44.44.44.44]) (user=test@sender.com mech=PLAIN bits=2) by abc.luxsci.com (8.13.7/8.13.7) with ESMTP id n141SAfo021855 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NOT) for <test@domain.com>; Tue, 3 Feb 2009 19:28:10 -0600", "Received": "from [192.168.0.3] (verizon.net [44.44.44.44]) (user=test@sender.com mech=PLAIN bits=2) by abc.luxsci.com (8.13.7/8.13.7) with ESMTP id n141SAfo021855 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NOT) for <test@domain.com>; Tue, 3 Feb 2009 19:28:10 -0600",
"X-Received":"by 192.168.0.45 with SMTP id q4mr156123401yw1g.911.1912342394963; Tue, 3 Feb 2009 19:32:15 -0600 (PST)", "X-Received": "by 192.168.0.45 with SMTP id q4mr156123401yw1g.911.1912342394963; Tue, 3 Feb 2009 19:32:15 -0600 (PST)",
"Message-ID":"<4988EF2D.40804@example.com>", "Message-ID": "<4988EF2D.40804@example.com>",
"Date":"Tue, 03 Feb 2009 20:28:13 -0500", "Date": "Tue, 03 Feb 2009 20:28:13 -0500",
"From":'"Innocent Person" <IgnoreMeImInnocent@sender.com>', "From": '"Innocent Person" <IgnoreMeImInnocent@sender.com>',
"User-Agent":'Thunderbird 2.0.0.19 (Windows/20081209)', "User-Agent": 'Thunderbird 2.0.0.19 (Windows/20081209)',
"Sender":'"Malicious MailAgent" <mailagent@example.com>', "Sender": '"Malicious MailAgent" <mailagent@example.com>',
"References":"<CI7DgL-A6dm92s7gf4-88g@E_0x238G4K2H08H9SDwsw8b6LwuA@mail.example.com>", "References": "<CI7DgL-A6dm92s7gf4-88g@E_0x238G4K2H08H9SDwsw8b6LwuA@mail.example.com>",
"In-Reply-To":"<CI7DgL-A6dm92s7gf4-88g@E_0x238G4K2H08H9SDwsw8b6LwuA@mail.example.com>", "In-Reply-To": "<CI7DgL-A6dm92s7gf4-88g@E_0x238G4K2H08H9SDwsw8b6LwuA@mail.example.com>",
"Accept-Language":'en-US', "Accept-Language": 'en-US',
"X-Mailer":'mlx 5.1.7', "X-Mailer": 'mlx 5.1.7',
"Return-Path": "evil_spoofer@example.com", "Return-Path": "evil_spoofer@example.com",
"Thread-Topic":'This is a thread.', "Thread-Topic": 'This is a thread.',
"Thread-Index":'AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==', "Thread-Index": 'AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==',
"Content-Language":'en-US', "Content-Language": 'en-US',
"To":'"Testy Testerson" <test@domain.com>', "To": '"Testy Testerson" <test@domain.com>',
"Cc":'"Second Person" <second@domain.com>, "Other Friend" <other@friend.net>, "Last One" <last_one@finally.com>', "Cc": '"Second Person" <second@domain.com>, "Other Friend" <other@friend.net>, "Last One" <last_one@finally.com>',
"Subject":'Example Message', "Subject": 'Example Message',
"MIME-Version":'1.0'} "MIME-Version": '1.0'}
msg = MIMEMultipart() msg = MIMEMultipart()
for key, val in headers.items(): for key, val in headers.items():
msg.add_header(key, val) msg.add_header(key, val)