From c508e60f65cc113371b6e7b358da7a86a21e3ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 27 Feb 2017 13:32:31 +0100 Subject: [PATCH] Add OpenIOC import module --- REQUIREMENTS | 1 + misp_modules/modules/import_mod/__init__.py | 3 +- .../modules/import_mod/openiocimport.py | 58 ++++++++++++ tests/openioc.xml | 91 +++++++++++++++++++ tests/test.py | 85 +++++++++-------- 5 files changed, 200 insertions(+), 38 deletions(-) create mode 100755 misp_modules/modules/import_mod/openiocimport.py create mode 100644 tests/openioc.xml diff --git a/REQUIREMENTS b/REQUIREMENTS index 6cda15a6..b4e5ebc9 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -19,3 +19,4 @@ pytesseract SPARQLWrapper domaintools_api pygeoip +bs4 diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index fd5d5394..6beeaa28 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,3 +1,4 @@ from . import _vmray -__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', 'email_import', 'mispjson'] +__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', + 'email_import', 'mispjson', 'openiocimport'] diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py new file mode 100755 index 00000000..27ef3f9c --- /dev/null +++ b/misp_modules/modules/import_mod/openiocimport.py @@ -0,0 +1,58 @@ +import json +import base64 + +from pymisp.tools import openioc + +misperrors = {'error': 'Error'} +userConfig = {} +inputSource = ['file'] + +moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', + 'description': 'Import OpenIOC package', + 'module-type': ['import']} + +moduleconfig = [] + + +def handler(q=False): + # Just in case we have no data + if q is False: + return False + + # The return value + r = {'results': []} + + # Load up that JSON + q = json.loads(q) + + # It's b64 encoded, so decode that stuff + package = base64.b64decode(q.get("data")).decode('utf-8') + + # If something really weird happened + if not package: + return json.dumps({"success": 0}) + + pkg = openioc.load_openioc(package) + for attrib in pkg.attributes: + r["results"].append({"values": [attrib.value], "types": [attrib.type], "categories": [attrib.category]}) + return r + + +def introspection(): + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/tests/openioc.xml b/tests/openioc.xml new file mode 100644 index 00000000..dc7858ce --- /dev/null +++ b/tests/openioc.xml @@ -0,0 +1,91 @@ + + + STUXNET VIRUS (METHODOLOGY) + Generic indicator for the stuxnet virus. When loaded, stuxnet spawns lsass.exe in a suspended state. The malware then maps in its own executable section and fixes up the CONTEXT to point to the newly mapped in section. This is a common task performed by malware and allows the malware to execute under the pretense of a known and trusted process. + methodology + Mandiant + 0001-01-01T00:00:00 + + + + + + .stub + + + + mdmcpq3.PNF + + + + mdmeric3.PNF + + + + oem6C.PNF + + + + oem7A.PNF + + + + + fs_rec.sys + + + + mrxsmb.sys + + + + sr.sys + + + + fastfat.sys + + + + + + mrxcls.sys + + + + Realtek Semiconductor Corp + + + + + + mrxnet.sys + + + + Realtek Semiconductor Corp + + + + + + HKEY_LOCAL_MACHINE\SYSTEM\ControlSet001\Services\MRxCls\ImagePath + + + + mrxcls.sys + + + + + + HKEY_LOCAL_MACHINE\SYSTEM\ControlSet001\Services\MRxNet\ImagePath + + + + mrxnet.sys + + + + + \ No newline at end of file diff --git a/tests/test.py b/tests/test.py index a94bbdf6..156846e6 100644 --- a/tests/test.py +++ b/tests/test.py @@ -41,6 +41,20 @@ class TestModules(unittest.TestCase): print(response.json()) response.connection.close() + def test_openioc(self): + with open("tests/openioc.xml", "rb") as f: + content = base64.b64encode(f.read()) + data = json.dumps({"module": "openiocimport", + "data": content.decode(), + }) + response = requests.post(self.url + "query", data=data).json() + print(response) + + print("OpenIOC :: {}".format(response)) + values = [x["values"][0] for x in response["results"]] + assert("mrxcls.sys" in values) + assert("mdmcpq3.PNF" in values) + def test_stix(self): with open("tests/stix.xml", "rb") as f: content = base64.b64encode(f.read()) @@ -57,7 +71,7 @@ class TestModules(unittest.TestCase): assert("eu-society.com" in values) def test_email_headers(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} @@ -105,7 +119,7 @@ class TestModules(unittest.TestCase): self.assertIn("", values) def test_email_attachment_basic(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} @@ -128,9 +142,8 @@ class TestModules(unittest.TestCase): attch_data = base64.b64decode(i["data"]) self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') - def test_email_attachment_unpack(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": None, "extract_urls": None} @@ -162,7 +175,7 @@ class TestModules(unittest.TestCase): def test_email_dont_unpack_compressed_doc_attachments(self): """Ensures that compressed """ - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": None, "extract_urls": None} @@ -192,9 +205,8 @@ class TestModules(unittest.TestCase): self.assertEqual(filesum.hexdigest(), '098da5381a90d4a51e6b844c18a0fecf2e364813c2f8b317cfdc51c21f2506a5') - def test_email_attachment_unpack_with_password(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -221,9 +233,8 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') - def test_email_attachment_password_in_body(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -246,7 +257,7 @@ class TestModules(unittest.TestCase): 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') def test_email_attachment_password_in_body_quotes(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -274,7 +285,7 @@ class TestModules(unittest.TestCase): 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') def test_email_attachment_password_in_html_body(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -304,7 +315,7 @@ class TestModules(unittest.TestCase): query['data'] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) - #print(response.json()) + # print(response.json()) values = [x["values"] for x in response.json()["results"]] self.assertIn('EICAR.com', values) for i in response.json()['results']: @@ -315,7 +326,7 @@ class TestModules(unittest.TestCase): 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') def test_email_attachment_password_in_subject(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -344,9 +355,8 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') - def test_email_extract_html_body_urls(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": "true"} @@ -374,12 +384,12 @@ without modifying core components. The API is available via a simple REST API wh query['data'] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) - #print(response.json()) + # print(response.json()) values = [x["values"] for x in response.json()["results"]] self.assertIn("https://github.com/MISP/MISP", values) self.assertIn("https://www.circl.lu/assets/files/misp-training/3.1-MISP-modules.pdf", values) - #def test_domaintools(self): + # def test_domaintools(self): # query = {'config': {'username': 'test_user', 'api_key': 'test_key'}, 'module': 'domaintools', 'domain': 'domaintools.com'} # try: # response = requests.post(self.url + "query", data=json.dumps(query)).json() @@ -388,33 +398,34 @@ without modifying core components. The API is available via a simple REST API wh # response = requests.post(self.url + "query", data=json.dumps(query)).json() # print(response) + def decode_email(message): message64 = base64.b64encode(message.as_bytes()).decode() return message64 def get_base_email(): - headers = {"Received":"via dmail-2008.19 for +INBOX; Tue, 3 Feb 2009 19:29:12 -0600 (CST)", - "Received":"from abc.luxsci.com ([10.10.10.10]) by xyz.luxsci.com (8.13.7/8.13.7) with ESMTP id n141TCa7022588 for ; Tue, 3 Feb 2009 19:29:12 -0600", - "Received":"from [192.168.0.3] (verizon.net [44.44.44.44]) (user=test@sender.com mech=PLAIN bits=2) by abc.luxsci.com (8.13.7/8.13.7) with ESMTP id n141SAfo021855 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NOT) for ; Tue, 3 Feb 2009 19:28:10 -0600", - "X-Received":"by 192.168.0.45 with SMTP id q4mr156123401yw1g.911.1912342394963; Tue, 3 Feb 2009 19:32:15 -0600 (PST)", - "Message-ID":"<4988EF2D.40804@example.com>", - "Date":"Tue, 03 Feb 2009 20:28:13 -0500", - "From":'"Innocent Person" ', - "User-Agent":'Thunderbird 2.0.0.19 (Windows/20081209)', - "Sender":'"Malicious MailAgent" ', - "References":"", - "In-Reply-To":"", - "Accept-Language":'en-US', - "X-Mailer":'mlx 5.1.7', + headers = {"Received": "via dmail-2008.19 for +INBOX; Tue, 3 Feb 2009 19:29:12 -0600 (CST)", + "Received": "from abc.luxsci.com ([10.10.10.10]) by xyz.luxsci.com (8.13.7/8.13.7) with ESMTP id n141TCa7022588 for ; Tue, 3 Feb 2009 19:29:12 -0600", + "Received": "from [192.168.0.3] (verizon.net [44.44.44.44]) (user=test@sender.com mech=PLAIN bits=2) by abc.luxsci.com (8.13.7/8.13.7) with ESMTP id n141SAfo021855 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NOT) for ; Tue, 3 Feb 2009 19:28:10 -0600", + "X-Received": "by 192.168.0.45 with SMTP id q4mr156123401yw1g.911.1912342394963; Tue, 3 Feb 2009 19:32:15 -0600 (PST)", + "Message-ID": "<4988EF2D.40804@example.com>", + "Date": "Tue, 03 Feb 2009 20:28:13 -0500", + "From": '"Innocent Person" ', + "User-Agent": 'Thunderbird 2.0.0.19 (Windows/20081209)', + "Sender": '"Malicious MailAgent" ', + "References": "", + "In-Reply-To": "", + "Accept-Language": 'en-US', + "X-Mailer": 'mlx 5.1.7', "Return-Path": "evil_spoofer@example.com", - "Thread-Topic":'This is a thread.', - "Thread-Index":'AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==', - "Content-Language":'en-US', - "To":'"Testy Testerson" ', - "Cc":'"Second Person" , "Other Friend" , "Last One" ', - "Subject":'Example Message', - "MIME-Version":'1.0'} + "Thread-Topic": 'This is a thread.', + "Thread-Index": 'AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==', + "Content-Language": 'en-US', + "To": '"Testy Testerson" ', + "Cc": '"Second Person" , "Other Friend" , "Last One" ', + "Subject": 'Example Message', + "MIME-Version": '1.0'} msg = MIMEMultipart() for key, val in headers.items(): msg.add_header(key, val)