From 83a9d695eaf220afacab16e5d8ab90e4ab246b6b Mon Sep 17 00:00:00 2001 From: seamus tuohy Date: Tue, 10 Jan 2017 09:46:04 -0500 Subject: [PATCH] Email import no longer unzips major compressed text document formats. Let this commit serve as a warning about the perils of duck typing. Word documents (docx,odt,etc) were being uncompressed when they were attached to emails. The email importer now checks a list of well known extensions and will not attempt to unzip them. It is stuck using a list of extensions instead of using file magic because many of these formats produce an application/zip mimetype when scanned. --- .../modules/import_mod/email_import.py | 41 ++++++++++++------ tests/test.py | 38 ++++++++++++++-- tests/test_files/test.docx | Bin 0 -> 4031 bytes 3 files changed, 61 insertions(+), 18 deletions(-) create mode 100644 tests/test_files/test.docx diff --git a/misp_modules/modules/import_mod/email_import.py b/misp_modules/modules/import_mod/email_import.py index 2027303..e7564f1 100644 --- a/misp_modules/modules/import_mod/email_import.py +++ b/misp_modules/modules/import_mod/email_import.py @@ -144,20 +144,33 @@ def handler(q=False): # Base attachment data is default attachment_files = [{"values": filename, "data": base64.b64encode(attachment_data).decode()}] if unzip is True: # Attempt to unzip the attachment and return its files - try: - attachment_files += get_zipped_contents(filename, attachment_data) - except RuntimeError: # File is encrypted with a password - if zip_pass_crack is True: - if password_list is None: - password_list = get_zip_passwords(message) - password = test_zip_passwords(attachment_data, password_list) - if password is None: # Inform the analyst that we could not crack password - attachment_files[0]['comment'] = "Encrypted Zip: Password could not be cracked from message" - else: - attachment_files[0]['comment'] = """Original Zipped Attachment with Password {0}""".format(password) - attachment_files += get_zipped_contents(filename, attachment_data, password=password) - except zipfile.BadZipFile: # Attachment is not a zipfile - attachment_files += [{"values": filename, "data": base64.b64encode(attachment_data).decode()}] + zipped_files = ["doc", "docx", "dot", "dotx", "xls", + "xlsx", "xlm", "xla", "xlc", "xlt", + "xltx", "xlw", "ppt", "pptx", "pps", + "ppsx", "pot", "potx", "potx", "sldx", + "odt", "ods", "odp", "odg", "odf", + "fodt", "fods", "fodp", "fodg", "ott", + "uot"] + + zipped_filetype = False + for ext in zipped_files: + if filename.endswith(ext) is True: + zipped_filetype = True + if zipped_filetype == False: + try: + attachment_files += get_zipped_contents(filename, attachment_data) + except RuntimeError: # File is encrypted with a password + if zip_pass_crack is True: + if password_list is None: + password_list = get_zip_passwords(message) + password = test_zip_passwords(attachment_data, password_list) + if password is None: # Inform the analyst that we could not crack password + attachment_files[0]['comment'] = "Encrypted Zip: Password could not be cracked from message" + else: + attachment_files[0]['comment'] = """Original Zipped Attachment with Password {0}""".format(password) + attachment_files += get_zipped_contents(filename, attachment_data, password=password) + except zipfile.BadZipFile: # Attachment is not a zipfile + pass for attch_item in attachment_files: attch_item["type"] = 'malware-sample' results.append(attch_item) diff --git a/tests/test.py b/tests/test.py index d15144d..a94bbdf 100644 --- a/tests/test.py +++ b/tests/test.py @@ -7,6 +7,7 @@ import base64 import json import io import zipfile +from hashlib import sha256 from email.mime.application import MIMEApplication from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart @@ -101,11 +102,7 @@ class TestModules(unittest.TestCase): self.assertEqual(types['email-x-mailer'], 1) self.assertIn("mlx 5.1.7", values) self.assertEqual(types['email-reply-to'], 1) - # The parser inserts a newline that I can't diagnose. - # It does not impact analysis since the interface strips it. - # But, I'm leaving this test failing self.assertIn("", values) - #self.assertIn("\n ", values) def test_email_attachment_basic(self): query = {"module":"email_import"} @@ -162,6 +159,39 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + def test_email_dont_unpack_compressed_doc_attachments(self): + """Ensures that compressed + """ + query = {"module":"email_import"} + query["config"] = {"unzip_attachments": "true", + "guess_zip_attachment_passwords": None, + "extract_urls": None} + message = get_base_email() + text = """I am a test e-mail""" + message.attach(MIMEText(text, 'plain')) + with open("tests/test_files/test.docx", "rb") as fp: + eicar_mime = MIMEApplication(fp.read(), 'zip') + eicar_mime.add_header('Content-Disposition', 'attachment', filename="test.docx") + message.attach(eicar_mime) + query['data'] = decode_email(message) + data = json.dumps(query) + response = requests.post(self.url + "query", data=data) + values = [x["values"] for x in response.json()["results"]] + self.assertIn('test.docx', values) + types = {} + for i in response.json()['results']: + types.setdefault(i["type"], 0) + types[i["type"]] += 1 + # Check that there is only one attachment in the bundle + self.assertEqual(types['malware-sample'], 1) + for i in response.json()['results']: + if i['type'] == 'malware-sample' and i["values"] == 'test.docx': + attch_data = base64.b64decode(i["data"]) + filesum = sha256() + filesum.update(attch_data) + self.assertEqual(filesum.hexdigest(), + '098da5381a90d4a51e6b844c18a0fecf2e364813c2f8b317cfdc51c21f2506a5') + def test_email_attachment_unpack_with_password(self): query = {"module":"email_import"} diff --git a/tests/test_files/test.docx b/tests/test_files/test.docx new file mode 100644 index 0000000000000000000000000000000000000000..df59d1222dac8545bedfd236d0c5244cfe9ece45 GIT binary patch literal 4031 zcmaJ^1z6L27pBG-A(N6)qy#}>q#`*&N@_G?^rT^gG#4pRknRo%iGk8`K}s4Yor*|^ z=p>|+{5J3BmFxFi&z|ksp67r5|DE@|=RNOFTb-DMj)07ejDU%WPmSPIP~g6syCa=E z1O@P)=mZUQVkqUk^&@(PwKq=u_j@tB9P+LlJHW7X?Aj0pC}NL3nS+uyX-DSuH2ZFC zu#Pxmj`K9LqN1RkW%f`v$Z!+H!Dm)lkj-LLu*py+ZcHuRl&@?qtiX~+HTHhjEo`Z% zklW)-_S;&p9w{r1nv!_8H}!jILBe^tLgNz-z3@MY>JC&6uZjCHRC9%0x@}f$T2*sh zfL_zrX4vA9Z>ifi2hn6oAB5eJfBtn!%!aP0@+)Yi@22r}qxyzq-Xi!-?-P4b*MNcM z9;acmg{YSWozpDax!USLvJXRqoYS~^rHKd#wEq>6v$!j~UEOU2J&>NB_Aa&_0zS@8 zN-^qZIv|vads!S`F2Lfx>K9@g36=@+#k4O!eKb|uwhH1;5!t#e8nMVvDq@uq+}G}| z6Ab;>$o64vekA9;B%~n6k1cJeodpo{5(^m*DGR63Wl;Ip*RCKoKB2bP_5s-rDz&+W z8Fq~S=E8#s?0=oFQrc}{s9*PVn3_Z*{ScxoG7j>HC_Llk_(Zm~Co_oD`Dmvii(F`b z-U9wP=;@KU{d0VcVxB?ZB3uQpaPNPwk@ZK7f3@Dm)%uPz(!~>B>0dO6*C6kJfW!B4 zI0om6^}+GQE{Z7r9Ri{3`gUm}?^cLE_B58SF}6XSN>C^u^qw z)sO=%Qaa*Gv-(7bmHl^3MLnhdi3p^$AMs`g6p#*iB)o z^qgRf;v2K~(j>(GQ9LTqw_9&DC^zhRjX1hFR+T8G*JnsY0Xq-1$_;CJ;szMtl*N6O z7K@ki&t>NZmY3RW!BaP+nZQ~A7XKz0Om5^2S)UyM9^CexX|nT#1O)oTzvCM94{$v^ zeVvfsr>RSCSaT5q4rr~qe`p2+-_WNgbW@^F)?W8)13G1CSihS}nPg9@lAYxOWVrL` zHTkPzjs2x1Yj&r*bgw2V*K*bNPi+pTaX)Sp2s1%qk6mOko`kVJCMq;Ydn!ir=k*h> z3C%teGZK-8L2B-S$V+ZY667vkC z?>Tq|9URxLIEcbYdC!HH7(5uZViGS#Sgv|KyY0^~+5FA5U{$VhO62%TgSZ)jn8ASI z7JW=?zEb>CBt-4zH2v6=?)to^hn=LeQvPhZ;Twl(L4GKWmz8L@(ObvV=+Q|>-TWE5 zMW;|iJfiaCHpe7vHs_pA*~!E#@4@p4Q_ipm%Xy7hWlae%gY3A#TRNF%7h1O#*_w=6 zJ+=vluLRB)SrD7P*mvJH}u8 zt6*8dAZHBWP{IcexMvw5WMfy*Om6G_o69uxnm@P%c`M=iOrRSq@U02!(zUnup+ zG8levHv5DSPu5~D-7hGd3XGEdj;w!+|7l4q(i?D@#RxLQD; zNO)MMK6dHQ6tYh8Ks z(Em}ibavcYtjU*ou63g z^jw}3j!BpZ26~e=N!E4HHm6-1kO61)Qj~^=U41}j$(8Xq@tcg#c3Wp5sxC^E zDivix^-$c|<1m`+%&nBb2g)NWS7l_S=Ag=a8f&tfuEW$Dj8DQrO*2^>Ri zn>Oh(y|Zyv6lSxkKA!dp!YxrCS+AL~;1N#fE&zVV9WYMlaJ12OcSU&!TB1<+9g-a@ z?br#SQ`$Qw=d7NAv0BT=(VM81$cZ!-W$F~#!n^k65<<7u%j@I(1YB>t?s;eCXS$t) zXa^!SLhAugCPvF5VTI7Snbd&TYVfo8fND}^&(OOu*x;O&JM)i1;Z#opnhmF}igra)8kn^a2O~(>+naq~>_X*cE6l#jyLDr3&h(4W z%uH{IfxX2YrA@YB<&{g^KDRzit0JVDk3XRFu_MWuiNaeWf!I%#mDS@4sUQCcx!*W* zRzqB?Cd0-I*1S6`vKO%NIy%_-vYAfEfywUorSRllW9k|XiBKF~|879=bM`k@)~@c~ z7jAUI0Dj>r?FA?;1ujVO0{B(Apa73%(u?)3$F)4Cj`bs5IBXvCxvLfi;(OwuZ7meQ%C zAaGD_b+wIW&eSjF>z^Pv~$!Z zkt#=D`^&Uy53`f9;8eRai%cNKlbr33Aq!+eBVk5d?_zx_o$E-Jm!jC!<3|EeG~mmx zD7$2Qt2lus04XQDX(P(^NsxEnQS$Zjue(7)LBx2pnV7+$S8-^Yk>FWR zNJK~Q^SE{T*nvN8{WDG-zJ5(S?d9HlK?f4$Rb zqlfp(KLdcXe&7GdKb-Tg`KNa;-VXi@D;!>b74o+^{5AQs`2G_GYT$p9PcQ3V(@*OW zUZH=6I_Q)J{a3Gl%|1Ov|9rSN`Kjz53ifOIXpzu_WW4|Y literal 0 HcmV?d00001