From 48f3f9eea0ea92fdae52851430a827c16e611352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 May 2018 10:53:40 +0200 Subject: [PATCH 01/50] fix: Add extension in case the attachment has no name. --- mail_to_misp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index fc3f8cf..daccc51 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -107,7 +107,7 @@ class Mail2MISP(): # Create file objects for the attachments for attachment_name, attachment in email_object.attachments: if not attachment_name: - attachment_name = 'NameMissing' + attachment_name = 'NameMissing.txt' f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) self.misp_event.add_object(f_object) if main_object: From 942bffd93adf924259b7311947e00c1551b778f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 May 2018 10:54:09 +0200 Subject: [PATCH 02/50] new: Add support for starttls --- fake_smtp.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/fake_smtp.py b/fake_smtp.py index d47b50e..17576b0 100755 --- a/fake_smtp.py +++ b/fake_smtp.py @@ -1,9 +1,21 @@ #!/usr/bin/env python import sys +import ssl from pathlib import Path import importlib from subprocess import run, PIPE -import aiosmtpd.controller +from aiosmtpd.controller import Controller +from aiosmtpd.smtp import SMTP +import subprocess + +key_path = Path('certs', 'key.pem') +cert_path = Path('certs', 'cert.pem') + + +# Pass SSL context to aiosmtpd +class ControllerStarttls(Controller): + def factory(self): + return SMTP(self.handler, require_starttls=False, tls_context=context) class CustomSMTPHandler: @@ -29,10 +41,16 @@ if __name__ == '__main__': smtp_port = config.smtp_port binpath = config.binpath + if not cert_path.exists() and not key_path.exists(): + subprocess.call(f'openssl req -x509 -newkey rsa:4096 -keyout {key_path.as_posix()} -out {cert_path.as_posix()} -days 365 -nodes -subj "/CN=localhost"', shell=True) + + context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + context.load_cert_chain(cert_path.as_posix(), key_path.as_posix()) + print("Starting Fake-SMTP-to-MISP server") handler = CustomSMTPHandler() - server = aiosmtpd.controller.Controller(handler, hostname=smtp_addr, port=smtp_port) + server = ControllerStarttls(handler, hostname=smtp_addr, port=smtp_port) server.start() input("Server started. Press Return to quit.") server.stop() From beace3732a19efb2a0a4d451e9fb8191c01852ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 May 2018 11:22:52 +0200 Subject: [PATCH 03/50] new: Add support for STARTTLS and SSL --- certs/.keepdir | 0 fake_smtp.py | 62 ++++++++++++++++++++++++++----------- fake_smtp_config.py-example | 1 + 3 files changed, 45 insertions(+), 18 deletions(-) create mode 100644 certs/.keepdir diff --git a/certs/.keepdir b/certs/.keepdir new file mode 100644 index 0000000..e69de29 diff --git a/fake_smtp.py b/fake_smtp.py index 17576b0..dfc66bc 100755 --- a/fake_smtp.py +++ b/fake_smtp.py @@ -7,15 +7,30 @@ from subprocess import run, PIPE from aiosmtpd.controller import Controller from aiosmtpd.smtp import SMTP import subprocess +import argparse key_path = Path('certs', 'key.pem') cert_path = Path('certs', 'cert.pem') +def get_context(): + if not cert_path.exists() and not key_path.exists(): + subprocess.call(f'openssl req -x509 -newkey rsa:4096 -keyout {key_path.as_posix()} -out {cert_path.as_posix()} -days 365 -nodes -subj "/CN=localhost"', shell=True) + + context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + context.load_cert_chain(cert_path.as_posix(), key_path.as_posix()) + + # Pass SSL context to aiosmtpd -class ControllerStarttls(Controller): +class ControllerSSL(Controller): def factory(self): - return SMTP(self.handler, require_starttls=False, tls_context=context) + return SMTP(self.handler, ssl_context=get_context()) + + +# Pass SSL context to aiosmtpd +class ControllerSTARTTLS(Controller): + def factory(self): + return SMTP(self.handler, require_starttls=False, tls_context=get_context()) class CustomSMTPHandler: @@ -30,27 +45,38 @@ class CustomSMTPHandler: if __name__ == '__main__': - configmodule = Path(__file__).as_posix().replace('.py', '_config') - if Path(f'{configmodule}.py').exists(): - config = importlib.import_module(configmodule) + parser = argparse.ArgumentParser(description='Launch a fake SMTP server to push SPAMs to a MISP instance') + parser.add_argument("--path", default='./mail_to_misp.py', help="Path to the mail_to_misp.py script.") + parser.add_argument("--host", default='127.0.0.1', help="IP to attach the SMTP server to.") + parser.add_argument("--port", default='2525', help="Port of the SMTP server") + parser.add_argument("--ssl", action='store_true', help="Pure SMTPs.") + args = parser.parse_args() + + if not args.path and not args.host and not args.port and not args.ssl: + configmodule = Path(__file__).as_posix().replace('.py', '_config') + if Path(f'{configmodule}.py').exists(): + config = importlib.import_module(configmodule) + else: + print("Couldn't locate config file {0}".format(f'{configmodule}.py')) + sys.exit(-1) + + binpath = config.binpath + smtp_addr = config.smtp_addr + smtp_port = config.smtp_port + smtps = config.ssl else: - print("Couldn't locate config file {0}".format(f'{configmodule}.py')) - sys.exit(-1) - - smtp_addr = config.smtp_addr - smtp_port = config.smtp_port - binpath = config.binpath - - if not cert_path.exists() and not key_path.exists(): - subprocess.call(f'openssl req -x509 -newkey rsa:4096 -keyout {key_path.as_posix()} -out {cert_path.as_posix()} -days 365 -nodes -subj "/CN=localhost"', shell=True) - - context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - context.load_cert_chain(cert_path.as_posix(), key_path.as_posix()) + binpath = args.path + smtp_addr = args.host + smtp_port = args.port + smtps = args.ssl print("Starting Fake-SMTP-to-MISP server") handler = CustomSMTPHandler() - server = ControllerStarttls(handler, hostname=smtp_addr, port=smtp_port) + if smtps: + server = ControllerSSL(handler, hostname=smtp_addr, port=smtp_port) + else: + server = ControllerSTARTTLS(handler, hostname=smtp_addr, port=smtp_port) server.start() input("Server started. Press Return to quit.") server.stop() diff --git a/fake_smtp_config.py-example b/fake_smtp_config.py-example index 48b208a..1d111e0 100644 --- a/fake_smtp_config.py-example +++ b/fake_smtp_config.py-example @@ -6,3 +6,4 @@ binpath = Path(__file__).cwd() / 'mail_to_misp.py' smtp_addr = '127.0.0.1' smtp_port = 2525 +ssl = False From eb9b7d907de5ef2d66c578b077596fb469b94789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 May 2018 14:02:01 +0200 Subject: [PATCH 04/50] new: Support email forwarding --- fake_smtp.py | 29 ++++++++++++++++------------- fake_smtp_config.py-example | 5 ++++- mail_to_misp_forward.py | 1 + 3 files changed, 21 insertions(+), 14 deletions(-) create mode 120000 mail_to_misp_forward.py diff --git a/fake_smtp.py b/fake_smtp.py index dfc66bc..b9e97ef 100755 --- a/fake_smtp.py +++ b/fake_smtp.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -import sys import ssl from pathlib import Path import importlib @@ -9,11 +8,11 @@ from aiosmtpd.smtp import SMTP import subprocess import argparse -key_path = Path('certs', 'key.pem') -cert_path = Path('certs', 'cert.pem') - def get_context(): + key_path = Path('certs', 'key.pem') + cert_path = Path('certs', 'cert.pem') + if not cert_path.exists() and not key_path.exists(): subprocess.call(f'openssl req -x509 -newkey rsa:4096 -keyout {key_path.as_posix()} -out {cert_path.as_posix()} -days 365 -nodes -subj "/CN=localhost"', shell=True) @@ -39,7 +38,10 @@ class CustomSMTPHandler: print(f'Message addressed from: {envelope.mail_from}') print(f'Message addressed to : {envelope.rcpt_tos}') print(f'Message length : {len(envelope.content)}') - p = run([binpath, "-"], stdout=PIPE, input=envelope.content) + if email_forward in envelope.rcpt_tos: + p = run([binpath_forward, "-"], stdout=PIPE, input=envelope.content) + else: + p = run([binpath, "-"], stdout=PIPE, input=envelope.content) print(p) return '250 OK' @@ -47,25 +49,26 @@ class CustomSMTPHandler: if __name__ == '__main__': parser = argparse.ArgumentParser(description='Launch a fake SMTP server to push SPAMs to a MISP instance') parser.add_argument("--path", default='./mail_to_misp.py', help="Path to the mail_to_misp.py script.") + parser.add_argument("--path_forward", default='./mail_to_misp.py', help="Path to the mail_to_misp.py script.") + parser.add_argument("--email_forward", default='mail2misp@example.com', help="Path to the mail_to_misp.py script.") parser.add_argument("--host", default='127.0.0.1', help="IP to attach the SMTP server to.") parser.add_argument("--port", default='2525', help="Port of the SMTP server") parser.add_argument("--ssl", action='store_true', help="Pure SMTPs.") args = parser.parse_args() - if not args.path and not args.host and not args.port and not args.ssl: - configmodule = Path(__file__).as_posix().replace('.py', '_config') - if Path(f'{configmodule}.py').exists(): - config = importlib.import_module(configmodule) - else: - print("Couldn't locate config file {0}".format(f'{configmodule}.py')) - sys.exit(-1) - + configmodule = Path(__file__).as_posix().replace('.py', '_config') + if Path(f'{configmodule}.py').exists(): + config = importlib.import_module(configmodule) binpath = config.binpath + binpath_forward = config.binpath_forward + email_forward = config.email_forward smtp_addr = config.smtp_addr smtp_port = config.smtp_port smtps = config.ssl else: binpath = args.path + binpath_forward = args.path_forward + email_forward = args.email_forward smtp_addr = args.host smtp_port = args.port smtps = args.ssl diff --git a/fake_smtp_config.py-example b/fake_smtp_config.py-example index 1d111e0..54f19c1 100644 --- a/fake_smtp_config.py-example +++ b/fake_smtp_config.py-example @@ -2,7 +2,10 @@ # -*- coding: utf-8 -*- from pathlib import Path -binpath = Path(__file__).cwd() / 'mail_to_misp.py' +binpath = Path(__file__).parent / 'mail_to_misp.py' +binpath_forward = Path(__file__).parent / 'mail_to_misp_forward.py' + +forward_email = 'mail2misp@example.com' smtp_addr = '127.0.0.1' smtp_port = 2525 diff --git a/mail_to_misp_forward.py b/mail_to_misp_forward.py new file mode 120000 index 0000000..b6e0429 --- /dev/null +++ b/mail_to_misp_forward.py @@ -0,0 +1 @@ +mail_to_misp.py \ No newline at end of file From 5e91bdce6dcb296e527d0538f17ae95f5ba1ea37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 May 2018 15:22:28 +0200 Subject: [PATCH 05/50] fix: Properly parse the email bodies. --- mail_to_misp.py | 102 +++++++++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 45 deletions(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index daccc51..69ba6a3 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -37,8 +37,8 @@ class Mail2MISP(): def __init__(self, misp_url, misp_key, verifycert, config): self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug) - self.debug = config.debug self.config = config + self.debug = self.config.debug # Init Faup self.f = Faup() @@ -53,10 +53,9 @@ class Mail2MISP(): # Initialize the MISP event self.misp_event = MISPEvent() self.misp_event.info = f'{config.email_subject_prefix} - {self.subject}' - self.misp_event.distribution = config.m2m_auto_distribution + self.misp_event.distribution = self.config.m2m_auto_distribution self.misp_event.threat_level_id = 3 self.misp_event.analysis = 1 - self.misp_event.add_tag(config.tlptag_default) def sighting(self, value, source): '''Add a sighting''' @@ -66,19 +65,20 @@ class Mail2MISP(): def _find_inline_forward(self): '''Does the body contains a forwarded email?''' - for identifier in config.forward_identifiers: + for identifier in self.config.forward_identifiers: if identifier in self.clean_email_body: self.clean_email_body, fw_email = self.clean_email_body.split(identifier) - self.forwarded_email(pseudofile=BytesIO(fw_email.encode())) + return self.forwarded_email(pseudofile=BytesIO(fw_email.encode())) def _find_attached_forward(self): + forwarded_emails = [] for attachment in self.original_mail.iter_attachments(): # Search for email forwarded as attachment # I could have more than one, attaching everything. if attachment.get_filename() and attachment.get_filename().endswith('.eml'): - self.forwarded_email(pseudofile=BytesIO(attachment.get_content().as_bytes())) + forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content().as_bytes()))) else: - if self.config_from_email_body.get('attachment') == config.m2m_benign_attachment_keyword: + if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword: # Attach sane file self.misp_event.add_attribute('attachment', value='Report', data=BytesIO(attachment.get_content().as_bytes())) @@ -89,11 +89,12 @@ class Mail2MISP(): if main_object: self.misp_event.add_object(main_object) [self.misp_event.add_object(section) for section in sections] + return forwarded_emails def email_from_spamtrap(self): '''The email comes from a spamtrap and should be attached as-is.''' self.clean_email_body = self.original_mail.get_body().as_string() - self.forwarded_email(self.pseudofile) + return self.forwarded_email(self.pseudofile) def forwarded_email(self, pseudofile: BytesIO): '''Extracts all possible indicators out of an email and create a MISP event out of it. @@ -117,6 +118,7 @@ class Mail2MISP(): email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment') self.process_body_iocs(email_object) self.misp_event.add_object(email_object) + return email_object def process_email_body(self): self.clean_email_body = self.original_mail.get_body().as_string() @@ -128,33 +130,36 @@ class Mail2MISP(): self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '', self.original_mail.get_body().as_string(), flags=re.MULTILINE) - self._find_inline_forward() - self._find_attached_forward() - # # Prepare extraction of IOCs - # Refang email data - self.clean_email_body = refang(self.clean_email_body) - - # Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message - for ignoreline in config.ignorelist: - self.clean_email_body = re.sub(rf'^{ignoreline}.*\n?', '', self.clean_email_body, flags=re.MULTILINE) - # Check if autopublish key is present and valid - if self.config_from_email_body.get('m2mkey') == config.m2m_key: + if self.config_from_email_body.get('m2mkey') == self.config.m2m_key: self.misp_event.publish() - # Add tags to the event if keywords are found in the mail - for tag in config.tlptags: - if any(alternativetag in self.clean_email_body for alternativetag in config.tlptags[tag]): - self.misp_event.add_tag(tag) - - # Remove everything after the stopword from the body - self.clean_email_body = self.clean_email_body.split(config.stopword, 1)[0] + self._find_inline_forward() + self._find_attached_forward() def process_body_iocs(self, email_object=None): if email_object: body = email_object.email.get_body().as_string() else: body = self.clean_email_body + + # Cleanup body content + # Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message + for ignoreline in self.config.ignorelist: + body = re.sub(rf'^{ignoreline}.*\n?', '', body, flags=re.MULTILINE) + + # Remove everything after the stopword from the body + self.clean_email_body = body.split(config.stopword, 1)[0] + + # Add tags to the event if keywords are found in the mail + for tag in self.config.tlptags: + if any(alternativetag in self.clean_email_body.lower() for alternativetag in self.config.tlptags[tag]): + self.misp_event.add_tag(tag) + + # Prepare extraction of IOCs + # Refang email data + body = refang(body) + # Extract and add hashes contains_hash = False for h in set(re.findall(hashmarker.MD5_REGEX, body)): @@ -162,25 +167,25 @@ class Mail2MISP(): attribute = self.misp_event.add_attribute('md5', h, enforceWarninglist=config.enforcewarninglist) if email_object: email_object.add_reference(attribute.uuid, 'contains') - if config.sighting: - self.sighting(h, config.sighting_source) + if self.config.sighting: + self.sighting(h, self.config.sighting_source) for h in set(re.findall(hashmarker.SHA1_REGEX, body)): contains_hash = True attribute = self.misp_event.add_attribute('sha1', h, enforceWarninglist=config.enforcewarninglist) if email_object: email_object.add_reference(attribute.uuid, 'contains') - if config.sighting: - self.sighting(h, config.sighting_source) + if self.config.sighting: + self.sighting(h, self.config.sighting_source) for h in set(re.findall(hashmarker.SHA256_REGEX, body)): contains_hash = True attribute = self.misp_event.add_attribute('sha256', h, enforceWarninglist=config.enforcewarninglist) if email_object: email_object.add_reference(attribute.uuid, 'contains') - if config.sighting: - self.sighting(h, config.sighting_source) + if self.config.sighting: + self.sighting(h, self.config.sighting_source) if contains_hash: - [self.misp_event.add_tag(tag) for tag in config.hash_only_tags] + [self.misp_event.add_tag(tag) for tag in self.config.hash_only_tags] # # Extract network IOCs urllist = [] @@ -197,7 +202,7 @@ class Mail2MISP(): self.f.decode(entry) domainname = self.f.get_domain().decode() - if domainname in config.excludelist: + if domainname in self.config.excludelist: # Ignore the entry continue @@ -214,19 +219,19 @@ class Mail2MISP(): if debug: syslog.syslog(domainname) - if domainname in config.internallist: # Add link to internal reference + if domainname in self.config.internallist: # Add link to internal reference attribute = self.misp_event.add_attribute('link', entry, category='Internal reference', to_ids=False, enforceWarninglist=False) if email_object: email_object.add_reference(attribute.uuid, 'contains') - elif domainname in config.externallist: # External analysis + elif domainname in self.config.externallist: # External analysis attribute = self.misp_event.add_attribute('link', entry, category='External analysis', to_ids=False, enforceWarninglist=False) if email_object: email_object.add_reference(attribute.uuid, 'contains') else: # The URL is probably an indicator. comment = "" - if (domainname in config.noidsflaglist) or (hostname in config.noidsflaglist): + if (domainname in self.config.noidsflaglist) or (hostname in self.config.noidsflaglist): ids_flag = False comment = "Known host (mostly for connectivity test or IP lookup)" if debug: @@ -249,16 +254,16 @@ class Mail2MISP(): enforceWarninglist=config.enforcewarninglist, comment=comment) if email_object: email_object.add_reference(attribute.uuid, 'contains') - if config.sighting: - self.sighting(entry, config.sighting_source) + if self.config.sighting: + self.sighting(entry, self.config.sighting_source) if hostname in hostname_processed: # Hostname already processed. continue hostname_processed.append(hostname) - if config.sighting: - self.sighting(hostname, config.sighting_source) + if self.config.sighting: + self.sighting(hostname, self.config.sighting_source) if debug: syslog.syslog(hostname) @@ -309,16 +314,23 @@ class Mail2MISP(): # Add additional tags depending on others tags = [] for tag in [t.name for t in self.misp_event.tags]: - if config.dependingtags.get(tag): - tags += config.dependingtags.get(tag) + if self.config.dependingtags.get(tag): + tags += self.config.dependingtags.get(tag) # Add additional tags according to configuration - for malware in config.malwaretags: + for malware in self.config.malwaretags: if malware.lower() in self.subject.lower(): - tags += config.malwaretags.get(malware) + tags += self.config.malwaretags.get(malware) if tags: [self.misp_event.add_tag(tag) for tag in tags] + has_tlp_tag = False + for tag in [t.name for t in self.misp_event.tags]: + if tag.lower().startswith('tlp'): + has_tlp_tag = True + if not has_tlp_tag: + self.misp_event.add_tag(config.tlptag_default) + self.misp.add_event(self.misp_event) From f1ba50a24cc32e549fb04c3eefcc320dcf5dd0e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 May 2018 15:25:47 +0200 Subject: [PATCH 06/50] fix: use proper body --- mail_to_misp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index 69ba6a3..851126a 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -153,7 +153,7 @@ class Mail2MISP(): # Add tags to the event if keywords are found in the mail for tag in self.config.tlptags: - if any(alternativetag in self.clean_email_body.lower() for alternativetag in self.config.tlptags[tag]): + if any(alternativetag in body.lower() for alternativetag in self.config.tlptags[tag]): self.misp_event.add_tag(tag) # Prepare extraction of IOCs From c978bec48c8f76b0376a326c3bb102b208faa83a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 May 2018 15:48:48 +0200 Subject: [PATCH 07/50] fix: typo --- mail_to_misp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index 851126a..46de20c 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -149,7 +149,7 @@ class Mail2MISP(): body = re.sub(rf'^{ignoreline}.*\n?', '', body, flags=re.MULTILINE) # Remove everything after the stopword from the body - self.clean_email_body = body.split(config.stopword, 1)[0] + body = body.split(self.config.stopword, 1)[0] # Add tags to the event if keywords are found in the mail for tag in self.config.tlptags: @@ -283,6 +283,7 @@ class Mail2MISP(): else: related_ips = [] try: + syslog.syslog(hostname) for rdata in dns.resolver.query(hostname, 'A'): if debug: syslog.syslog(str(rdata)) From 1f3e95ca676f778f523035c38a15ae8301f3d905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 May 2018 18:11:19 +0200 Subject: [PATCH 08/50] fix: Decode HTML content --- mail_to_misp.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index 46de20c..d800aa3 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -7,6 +7,7 @@ import argparse import re import syslog from pathlib import Path +import html from io import BytesIO from ipaddress import ip_address from email import message_from_bytes, policy @@ -76,15 +77,16 @@ class Mail2MISP(): # Search for email forwarded as attachment # I could have more than one, attaching everything. if attachment.get_filename() and attachment.get_filename().endswith('.eml'): - forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content().as_bytes()))) + forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content()))) else: if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword: # Attach sane file - self.misp_event.add_attribute('attachment', value='Report', - data=BytesIO(attachment.get_content().as_bytes())) + attachment_filename = attachment.get_filename() + if not attachment_filename: + attachment_filename = 'Report.data' + self.misp_event.add_attribute('attachment', value=attachment_filename, data=BytesIO(attachment.get_content())) else: - f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content()), - filename=attachment.get_filename(), standalone=False) + f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content()), filename=attachment.get_filename(), standalone=False) self.misp_event.add_object(f_object) if main_object: self.misp_event.add_object(main_object) @@ -93,7 +95,7 @@ class Mail2MISP(): def email_from_spamtrap(self): '''The email comes from a spamtrap and should be attached as-is.''' - self.clean_email_body = self.original_mail.get_body().as_string() + self.clean_email_body = html.unescape(self.original_mail.get_body().get_payload(decode=True).decode()) return self.forwarded_email(self.pseudofile) def forwarded_email(self, pseudofile: BytesIO): @@ -121,14 +123,14 @@ class Mail2MISP(): return email_object def process_email_body(self): - self.clean_email_body = self.original_mail.get_body().as_string() + self.clean_email_body = html.unescape(self.original_mail.get_body().get_payload(decode=True).decode()) # Check if there are config lines in the body & convert them to a python dictionary: # :: => {: } self.config_from_email_body = {k: v for k, v in re.findall(f'{config.body_config_prefix}:(.*):(.*)', self.clean_email_body)} if self.config_from_email_body: # ... remove the config lines from the body self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '', - self.original_mail.get_body().as_string(), flags=re.MULTILINE) + html.unescape(self.original_mail.get_body().get_payload(decode=True).decode()), flags=re.MULTILINE) # Check if autopublish key is present and valid if self.config_from_email_body.get('m2mkey') == self.config.m2m_key: @@ -139,7 +141,7 @@ class Mail2MISP(): def process_body_iocs(self, email_object=None): if email_object: - body = email_object.email.get_body().as_string() + body = html.unescape(email_object.email.get_body().get_payload(decode=True).decode()) else: body = self.clean_email_body From d9de8f7344c9c8dc8d8e4fb87dbee3fbc2e13751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 May 2018 18:15:40 +0200 Subject: [PATCH 09/50] fix: last commit --- mail_to_misp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index d800aa3..3c54a3b 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -77,7 +77,7 @@ class Mail2MISP(): # Search for email forwarded as attachment # I could have more than one, attaching everything. if attachment.get_filename() and attachment.get_filename().endswith('.eml'): - forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content()))) + forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content().get_payload(decode=True)))) else: if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword: # Attach sane file From dd0d127f8c3b5e477c7efd625bef7283e06da9df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 May 2018 18:22:14 +0200 Subject: [PATCH 10/50] fix: Mail body can be empty... --- mail_to_misp.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index 3c54a3b..60a8a4c 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -123,20 +123,22 @@ class Mail2MISP(): return email_object def process_email_body(self): - self.clean_email_body = html.unescape(self.original_mail.get_body().get_payload(decode=True).decode()) - # Check if there are config lines in the body & convert them to a python dictionary: - # :: => {: } - self.config_from_email_body = {k: v for k, v in re.findall(f'{config.body_config_prefix}:(.*):(.*)', self.clean_email_body)} - if self.config_from_email_body: - # ... remove the config lines from the body - self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '', - html.unescape(self.original_mail.get_body().get_payload(decode=True).decode()), flags=re.MULTILINE) + mail_as_bytes = self.original_mail.get_body().get_payload(decode=True) + if mail_as_bytes: + self.clean_email_body = html.unescape(mail_as_bytes.decode()) + # Check if there are config lines in the body & convert them to a python dictionary: + # :: => {: } + self.config_from_email_body = {k: v for k, v in re.findall(f'{config.body_config_prefix}:(.*):(.*)', self.clean_email_body)} + if self.config_from_email_body: + # ... remove the config lines from the body + self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '', + html.unescape(self.original_mail.get_body().get_payload(decode=True).decode()), flags=re.MULTILINE) - # Check if autopublish key is present and valid - if self.config_from_email_body.get('m2mkey') == self.config.m2m_key: - self.misp_event.publish() + # Check if autopublish key is present and valid + if self.config_from_email_body.get('m2mkey') == self.config.m2m_key: + self.misp_event.publish() - self._find_inline_forward() + self._find_inline_forward() self._find_attached_forward() def process_body_iocs(self, email_object=None): From 65cd921a25d5a01ac253e0fbae1ff6b0dfab3891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 May 2018 19:35:34 +0200 Subject: [PATCH 11/50] fix: missing variable --- mail_to_misp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mail_to_misp.py b/mail_to_misp.py index 60a8a4c..da5417b 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -139,6 +139,8 @@ class Mail2MISP(): self.misp_event.publish() self._find_inline_forward() + else: + self.clean_email_body = '' self._find_attached_forward() def process_body_iocs(self, email_object=None): From 854452e610fbe3aa0853c9ae64e70a749f6f3516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 8 May 2018 10:28:48 +0200 Subject: [PATCH 12/50] fix: properly get body. --- mail_to_misp.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index da5417b..1ec06dc 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -79,14 +79,14 @@ class Mail2MISP(): if attachment.get_filename() and attachment.get_filename().endswith('.eml'): forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content().get_payload(decode=True)))) else: + filename = attachment.get_filename() + if not filename: + filename = 'missing_filename' if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword: # Attach sane file - attachment_filename = attachment.get_filename() - if not attachment_filename: - attachment_filename = 'Report.data' - self.misp_event.add_attribute('attachment', value=attachment_filename, data=BytesIO(attachment.get_content())) + self.misp_event.add_attribute('attachment', value=filename, data=BytesIO(attachment.get_content())) else: - f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content()), filename=attachment.get_filename(), standalone=False) + f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content()), filename=filename, standalone=False) self.misp_event.add_object(f_object) if main_object: self.misp_event.add_object(main_object) @@ -95,7 +95,7 @@ class Mail2MISP(): def email_from_spamtrap(self): '''The email comes from a spamtrap and should be attached as-is.''' - self.clean_email_body = html.unescape(self.original_mail.get_body().get_payload(decode=True).decode()) + self.clean_email_body = html.unescape(self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode()) return self.forwarded_email(self.pseudofile) def forwarded_email(self, pseudofile: BytesIO): @@ -123,7 +123,7 @@ class Mail2MISP(): return email_object def process_email_body(self): - mail_as_bytes = self.original_mail.get_body().get_payload(decode=True) + mail_as_bytes = self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True) if mail_as_bytes: self.clean_email_body = html.unescape(mail_as_bytes.decode()) # Check if there are config lines in the body & convert them to a python dictionary: @@ -132,7 +132,7 @@ class Mail2MISP(): if self.config_from_email_body: # ... remove the config lines from the body self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '', - html.unescape(self.original_mail.get_body().get_payload(decode=True).decode()), flags=re.MULTILINE) + html.unescape(self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode()), flags=re.MULTILINE) # Check if autopublish key is present and valid if self.config_from_email_body.get('m2mkey') == self.config.m2m_key: @@ -145,7 +145,7 @@ class Mail2MISP(): def process_body_iocs(self, email_object=None): if email_object: - body = html.unescape(email_object.email.get_body().get_payload(decode=True).decode()) + body = html.unescape(email_object.email.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode()) else: body = self.clean_email_body From a8c2e50036404acf4ad9d730bdf10bf9cf1d3623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 8 May 2018 11:03:38 +0200 Subject: [PATCH 13/50] fix: Properly pass the attached email to the object generator --- mail_to_misp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index 1ec06dc..35b1c3e 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -77,7 +77,7 @@ class Mail2MISP(): # Search for email forwarded as attachment # I could have more than one, attaching everything. if attachment.get_filename() and attachment.get_filename().endswith('.eml'): - forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content().get_payload(decode=True)))) + forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content().as_bytes()))) else: filename = attachment.get_filename() if not filename: From 864764abc42e410b1151dfa1038528f818396b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 8 May 2018 15:27:13 +0200 Subject: [PATCH 14/50] chg: Update requirements file --- reqirements.txt => requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) rename reqirements.txt => requirements.txt (77%) diff --git a/reqirements.txt b/requirements.txt similarity index 77% rename from reqirements.txt rename to requirements.txt index 3c73c27..f08e8d0 100644 --- a/reqirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ dnspython -pymisp[fileobjects] +lief +python-magic +git+https://github.com/MISP/PyMISP.git git+https://github.com/kbandla/pydeep.git git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python git+https://github.com/Rafiot/defang.git From f3bf811237a7822f183e5358c563d76f7fe605ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 8 May 2018 15:35:01 +0200 Subject: [PATCH 15/50] new: Add easy install --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 6e8018b..9144907 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,27 @@ Obviously, you would like to filter mails based on subject or from address and p ## Requirements +### The easy way + +```bash +# Install faup +git clone git://github.com/stricaud/faup.git +cd faup +mkdir build +cd build +cmake .. && make +sudo make install + +# Update Shared libs +sudo ldconfig + +# Install other python requirements +pip3 install -r requirements.txt + +# Test if the script is working +./mail_to_misp.py -h +``` + ### General - mail_to_misp requires access to a MISP instance (via API). From 1902ba8f7656a9d5143e16033b5f7eac7e5369d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 8 May 2018 15:41:08 +0200 Subject: [PATCH 16/50] chg: Make sure the scripts are started with python3. --- fake_smtp.py | 2 +- mail_to_misp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fake_smtp.py b/fake_smtp.py index b9e97ef..0e96e2e 100755 --- a/fake_smtp.py +++ b/fake_smtp.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import ssl from pathlib import Path import importlib diff --git a/mail_to_misp.py b/mail_to_misp.py index 35b1c3e..5373461 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import os From 311e86ab36314b890285839f8de281107db01aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 9 May 2018 10:47:24 +0200 Subject: [PATCH 17/50] fix: Typo in config file --- fake_smtp_config.py-example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fake_smtp_config.py-example b/fake_smtp_config.py-example index 54f19c1..a27a0bc 100644 --- a/fake_smtp_config.py-example +++ b/fake_smtp_config.py-example @@ -5,7 +5,7 @@ from pathlib import Path binpath = Path(__file__).parent / 'mail_to_misp.py' binpath_forward = Path(__file__).parent / 'mail_to_misp_forward.py' -forward_email = 'mail2misp@example.com' +email_forward = 'mail2misp@example.com' smtp_addr = '127.0.0.1' smtp_port = 2525 From fe0bdb8f23453d83355eba2dde5e3ff20a716940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 10 May 2018 09:53:18 +0200 Subject: [PATCH 18/50] fix: Do not fail if the email has no body. --- mail_to_misp.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index 5373461..087b8f3 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -95,7 +95,11 @@ class Mail2MISP(): def email_from_spamtrap(self): '''The email comes from a spamtrap and should be attached as-is.''' - self.clean_email_body = html.unescape(self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode()) + raw_body = self.original_mail.get_body(preferencelist=('html', 'plain')) + if raw_body: + self.clean_email_body = html.unescape(raw_body.get_payload(decode=True).decode()) + else: + self.clean_email_body = '' return self.forwarded_email(self.pseudofile) def forwarded_email(self, pseudofile: BytesIO): From b7d582927c8704af9d2a50c585a1de65d1566fc4 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Fri, 11 May 2018 09:23:41 +0200 Subject: [PATCH 19/50] reflected changes in alias file for new version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9144907..ba1f2a3 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ The implemented workflow is mainly for mail servers like Postfix. Client side im 1. Setup a new email address in the aliases file (e.g. /etc/aliases) and configure the correct path: -`misp_handler: "|/path/to/mail_to_misp.py"` +`misp_handler: "|/path/to/mail_to_misp.py -"` 2. Rebuild the DB: From 9d7e9922196700578269cf717469d51658887162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 11 May 2018 10:15:16 -0400 Subject: [PATCH 20/50] fix: Better decoding, update sample config --- mail_to_misp.py | 10 +++++----- mail_to_misp_config.py-example | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index 087b8f3..a5b0793 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -49,7 +49,7 @@ class Mail2MISP(): self.subject = self.original_mail.get('Subject') # Remove words from subject for removeword in self.config.removelist: - self.subject = re.sub(removeword, "", self.subject) + self.subject = re.sub(removeword, "", self.subject).strip() # Initialize the MISP event self.misp_event = MISPEvent() @@ -97,7 +97,7 @@ class Mail2MISP(): '''The email comes from a spamtrap and should be attached as-is.''' raw_body = self.original_mail.get_body(preferencelist=('html', 'plain')) if raw_body: - self.clean_email_body = html.unescape(raw_body.get_payload(decode=True).decode()) + self.clean_email_body = html.unescape(raw_body.get_payload(decode=True).decode('utf8', 'surrogateescape')) else: self.clean_email_body = '' return self.forwarded_email(self.pseudofile) @@ -129,14 +129,14 @@ class Mail2MISP(): def process_email_body(self): mail_as_bytes = self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True) if mail_as_bytes: - self.clean_email_body = html.unescape(mail_as_bytes.decode()) + self.clean_email_body = html.unescape(mail_as_bytes.decode('utf8', 'surrogateescape')) # Check if there are config lines in the body & convert them to a python dictionary: # :: => {: } self.config_from_email_body = {k: v for k, v in re.findall(f'{config.body_config_prefix}:(.*):(.*)', self.clean_email_body)} if self.config_from_email_body: # ... remove the config lines from the body self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '', - html.unescape(self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode()), flags=re.MULTILINE) + html.unescape(self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape')), flags=re.MULTILINE) # Check if autopublish key is present and valid if self.config_from_email_body.get('m2mkey') == self.config.m2m_key: @@ -149,7 +149,7 @@ class Mail2MISP(): def process_body_iocs(self, email_object=None): if email_object: - body = html.unescape(email_object.email.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode()) + body = html.unescape(email_object.email.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape')) else: body = self.clean_email_body diff --git a/mail_to_misp_config.py-example b/mail_to_misp_config.py-example index 8f8359e..916839f 100644 --- a/mail_to_misp_config.py-example +++ b/mail_to_misp_config.py-example @@ -39,8 +39,9 @@ enforcewarninglist = True sighting = True sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER" -# Remove "[tags]", "Re: ", "Fwd: " from subject -removelist = ("[\(\[].*?[\)\]]", "Re: ", "Fwd: ", "{Spam?} ") +# Remove "Re:", "Fwd:" and {Spam?} from subject +# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag] +removelist = ("Re:", "Fwd:", "\{Spam\?\} ") # TLP tag setup # Tuples contain different variations of spelling From 30037301560c3e135fccf6b49a9ec5eb0ae8b1c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 11 May 2018 11:33:58 -0400 Subject: [PATCH 21/50] fix: do not attach the full email by default unless spamtrap mode --- mail_to_misp.py | 4 +++- mail_to_misp_config.py-example | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index a5b0793..fc8b71c 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -40,6 +40,7 @@ class Mail2MISP(): self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug) self.config = config self.debug = self.config.debug + self.config_from_email_body = {} # Init Faup self.f = Faup() @@ -123,7 +124,8 @@ class Mail2MISP(): self.misp_event.add_object(section) email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment') self.process_body_iocs(email_object) - self.misp_event.add_object(email_object) + if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get('attach_original_mail'): + self.misp_event.add_object(email_object) return email_object def process_email_body(self): diff --git a/mail_to_misp_config.py-example b/mail_to_misp_config.py-example index 916839f..5f972cb 100644 --- a/mail_to_misp_config.py-example +++ b/mail_to_misp_config.py-example @@ -14,7 +14,7 @@ m2m_benign_attachment_keyword = 'benign' debug = False nameservers = ['149.13.33.69'] email_subject_prefix = 'M2M' -attach_original_mail = True +attach_original_mail = False excludelist = ('google.com', 'microsoft.com') externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net') From f208609812790e5b7dab6f6fa48ce300806a235a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 11 May 2018 14:50:19 -0400 Subject: [PATCH 22/50] new: Add more config options in the mail body. --- mail_to_misp.py | 15 +++++++++++---- mail_to_misp_config.py-example | 6 ++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index fc8b71c..accba6e 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -55,9 +55,9 @@ class Mail2MISP(): # Initialize the MISP event self.misp_event = MISPEvent() self.misp_event.info = f'{config.email_subject_prefix} - {self.subject}' - self.misp_event.distribution = self.config.m2m_auto_distribution - self.misp_event.threat_level_id = 3 - self.misp_event.analysis = 1 + self.misp_event.distribution = self.config.default_distribution + self.misp_event.threat_level_id = self.config.default_threat_level + self.misp_event.analysis = self.config.default_analysis def sighting(self, value, source): '''Add a sighting''' @@ -142,7 +142,14 @@ class Mail2MISP(): # Check if autopublish key is present and valid if self.config_from_email_body.get('m2mkey') == self.config.m2m_key: - self.misp_event.publish() + if self.config_from_email_body.get('distribution'): + self.misp_event.distribution = self.config_from_email_body.get('distribution') + if self.config_from_email_body.get('threat_level'): + self.misp_event.threat_level_id = self.config_from_email_body.get('threat_level') + if self.config_from_email_body.get('analysis'): + self.misp_event.analysis = self.config_from_email_body.get('analysis') + if self.config_from_email_body.get('publish'): + self.misp_event.publish() self._find_inline_forward() else: diff --git a/mail_to_misp_config.py-example b/mail_to_misp_config.py-example index 5f972cb..34fb226 100644 --- a/mail_to_misp_config.py-example +++ b/mail_to_misp_config.py-example @@ -4,11 +4,13 @@ misp_url = 'YOUR_MISP_URL' misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section misp_verifycert = True -body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs spamtrap = False +default_distribution = 0 +default_threat_level = 3 +default_analysis = 1 +body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs m2m_key = 'YOUSETYOURKEYHERE' -m2m_auto_distribution = '3' # 3 = All communities m2m_benign_attachment_keyword = 'benign' debug = False From fc17a40b184688e60f2c1b165ddd968096a5cab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 11 May 2018 15:20:21 -0400 Subject: [PATCH 23/50] chg: Strip spaces in the config --- mail_to_misp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index accba6e..04cce6e 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -134,7 +134,7 @@ class Mail2MISP(): self.clean_email_body = html.unescape(mail_as_bytes.decode('utf8', 'surrogateescape')) # Check if there are config lines in the body & convert them to a python dictionary: # :: => {: } - self.config_from_email_body = {k: v for k, v in re.findall(f'{config.body_config_prefix}:(.*):(.*)', self.clean_email_body)} + self.config_from_email_body = {k.strip(): v.strip() for k, v in re.findall(f'{config.body_config_prefix}:(.*):(.*)', self.clean_email_body)} if self.config_from_email_body: # ... remove the config lines from the body self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '', From c967c5e197063f82f055d117b8f689b157f2e447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 14 May 2018 17:23:30 -0400 Subject: [PATCH 24/50] new: Make it a lib, add test cases --- .gitmodules | 3 + .travis.yml | 35 +++ __init__.py | 0 mail2misp/__init__.py | 3 + hashmarker.py => mail2misp/hashmarker.py | 0 mail2misp/mail2misp.py | 350 +++++++++++++++++++++++ urlmarker.py => mail2misp/urlmarker.py | 0 mail_to_misp.py | 345 +--------------------- mail_to_misp_config.py-example | 2 +- tests/__init__.py | 0 tests/config_forward.py | 77 +++++ tests/config_spamtrap.py | 77 +++++ tests/mails | 1 + tests/tests.py | 57 ++++ 14 files changed, 605 insertions(+), 345 deletions(-) create mode 100644 .gitmodules create mode 100644 .travis.yml create mode 100644 __init__.py create mode 100644 mail2misp/__init__.py rename hashmarker.py => mail2misp/hashmarker.py (100%) mode change 100755 => 100644 create mode 100644 mail2misp/mail2misp.py rename urlmarker.py => mail2misp/urlmarker.py (100%) mode change 100755 => 100644 create mode 100644 tests/__init__.py create mode 100644 tests/config_forward.py create mode 100644 tests/config_spamtrap.py create mode 160000 tests/mails create mode 100644 tests/tests.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..1044271 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tests/mails"] + path = tests/mails + url = https://github.com/MISP/mail_to_misp_test.git diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..b3a5921 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,35 @@ +language: python + +cache: pip + +sudo: required + +addons: + apt: + sources: [ 'ubuntu-toolchain-r-test' ] + packages: + - libstdc++6 + - libfuzzy-dev + +python: + - "3.6" + - "3.6-dev" + +install: + - git clone git://github.com/stricaud/faup.git + - pushd faup/build + - cmake .. && make + - sudo make install + - popd + - sudo ldconfig + - pip install -U nose pip coverage + - pip install -U -r requirements.txt + - git submodule init + - git submodule update + +script: + - nosetests --with-coverage --cover-package=mail2misp tests/tests.py + +after_success: + - codecov + - coveralls diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mail2misp/__init__.py b/mail2misp/__init__.py new file mode 100644 index 0000000..cb5584d --- /dev/null +++ b/mail2misp/__init__.py @@ -0,0 +1,3 @@ +from . import urlmarker +from . import hashmarker +from .mail2misp import Mail2MISP diff --git a/hashmarker.py b/mail2misp/hashmarker.py old mode 100755 new mode 100644 similarity index 100% rename from hashmarker.py rename to mail2misp/hashmarker.py diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py new file mode 100644 index 0000000..d2218d8 --- /dev/null +++ b/mail2misp/mail2misp.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import re +import syslog +import html +from io import BytesIO +from ipaddress import ip_address +from email import message_from_bytes, policy + +from . import urlmarker +from . import hashmarker +from pyfaup.faup import Faup +from pymisp import PyMISP, MISPEvent, MISPObject, MISPSighting +from pymisp.tools import EMailObject, make_binary_objects +from defang import refang +import dns.resolver + + +def is_ip(address): + try: + ip_address(address) + except ValueError: + return False + return True + + +class Mail2MISP(): + + def __init__(self, misp_url, misp_key, verifycert, config, offline=False): + self.offline = offline + if not self.offline: + self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug) + self.config = config + self.debug = self.config.debug + self.config_from_email_body = {} + # Init Faup + self.f = Faup() + + def load_email(self, pseudofile): + self.pseudofile = pseudofile + self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default) + self.subject = self.original_mail.get('Subject') + # Remove words from subject + for removeword in self.config.removelist: + self.subject = re.sub(removeword, "", self.subject).strip() + + # Initialize the MISP event + self.misp_event = MISPEvent() + self.misp_event.info = f'{self.config.email_subject_prefix} - {self.subject}' + self.misp_event.distribution = self.config.default_distribution + self.misp_event.threat_level_id = self.config.default_threat_level + self.misp_event.analysis = self.config.default_analysis + + def sighting(self, value, source): + if self.offline: + raise Exception('The script is running in offline mode, ') + '''Add a sighting''' + s = MISPSighting() + s.from_dict(value=value, source=source) + self.misp.set_sightings(s) + + def _find_inline_forward(self): + '''Does the body contains a forwarded email?''' + for identifier in self.config.forward_identifiers: + if identifier in self.clean_email_body: + self.clean_email_body, fw_email = self.clean_email_body.split(identifier) + return self.forwarded_email(pseudofile=BytesIO(fw_email.encode())) + + def _find_attached_forward(self): + forwarded_emails = [] + for attachment in self.original_mail.iter_attachments(): + # Search for email forwarded as attachment + # I could have more than one, attaching everything. + if attachment.get_filename() and attachment.get_filename().endswith('.eml'): + forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content().as_bytes()))) + else: + filename = attachment.get_filename() + if not filename: + filename = 'missing_filename' + if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword: + # Attach sane file + self.misp_event.add_attribute('attachment', value=filename, data=BytesIO(attachment.get_content())) + else: + f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content()), filename=filename, standalone=False) + self.misp_event.add_object(f_object) + if main_object: + self.misp_event.add_object(main_object) + [self.misp_event.add_object(section) for section in sections] + return forwarded_emails + + def email_from_spamtrap(self): + '''The email comes from a spamtrap and should be attached as-is.''' + raw_body = self.original_mail.get_body(preferencelist=('html', 'plain')) + if raw_body: + self.clean_email_body = html.unescape(raw_body.get_payload(decode=True).decode('utf8', 'surrogateescape')) + else: + self.clean_email_body = '' + return self.forwarded_email(self.pseudofile) + + def forwarded_email(self, pseudofile: BytesIO): + '''Extracts all possible indicators out of an email and create a MISP event out of it. + * Gets all relevant Headers + * Attach the body + * Create MISP file objects (uses lief if possible) + * Set all references + ''' + email_object = EMailObject(pseudofile=pseudofile, attach_original_mail=True, standalone=False) + if email_object.attachments: + # Create file objects for the attachments + for attachment_name, attachment in email_object.attachments: + if not attachment_name: + attachment_name = 'NameMissing.txt' + f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) + self.misp_event.add_object(f_object) + if main_object: + self.misp_event.add_object(main_object) + for section in sections: + self.misp_event.add_object(section) + email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment') + self.process_body_iocs(email_object) + if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get('attach_original_mail'): + self.misp_event.add_object(email_object) + return email_object + + def process_email_body(self): + mail_as_bytes = self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True) + if mail_as_bytes: + self.clean_email_body = html.unescape(mail_as_bytes.decode('utf8', 'surrogateescape')) + # Check if there are config lines in the body & convert them to a python dictionary: + # :: => {: } + self.config_from_email_body = {k.strip(): v.strip() for k, v in re.findall(f'{self.config.body_config_prefix}:(.*):(.*)', self.clean_email_body)} + if self.config_from_email_body: + # ... remove the config lines from the body + self.clean_email_body = re.sub(rf'^{self.config.body_config_prefix}.*\n?', '', + html.unescape(self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape')), flags=re.MULTILINE) + # Check if autopublish key is present and valid + if self.config_from_email_body.get('m2mkey') == self.config.m2m_key: + if self.config_from_email_body.get('distribution'): + self.misp_event.distribution = self.config_from_email_body.get('distribution') + if self.config_from_email_body.get('threat_level'): + self.misp_event.threat_level_id = self.config_from_email_body.get('threat_level') + if self.config_from_email_body.get('analysis'): + self.misp_event.analysis = self.config_from_email_body.get('analysis') + if self.config_from_email_body.get('publish'): + self.misp_event.publish() + + self._find_inline_forward() + else: + self.clean_email_body = '' + self._find_attached_forward() + + def process_body_iocs(self, email_object=None): + if email_object: + body = html.unescape(email_object.email.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape')) + else: + body = self.clean_email_body + + # Cleanup body content + # Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message + for ignoreline in self.config.ignorelist: + body = re.sub(rf'^{ignoreline}.*\n?', '', body, flags=re.MULTILINE) + + # Remove everything after the stopword from the body + body = body.split(self.config.stopword, 1)[0] + + # Add tags to the event if keywords are found in the mail + for tag in self.config.tlptags: + if any(alternativetag in body.lower() for alternativetag in self.config.tlptags[tag]): + self.misp_event.add_tag(tag) + + # Prepare extraction of IOCs + # Refang email data + body = refang(body) + + # Extract and add hashes + contains_hash = False + for h in set(re.findall(hashmarker.MD5_REGEX, body)): + contains_hash = True + attribute = self.misp_event.add_attribute('md5', h, enforceWarninglist=self.config.enforcewarninglist) + if email_object: + email_object.add_reference(attribute.uuid, 'contains') + if self.config.sighting: + self.sighting(h, self.config.sighting_source) + for h in set(re.findall(hashmarker.SHA1_REGEX, body)): + contains_hash = True + attribute = self.misp_event.add_attribute('sha1', h, enforceWarninglist=self.config.enforcewarninglist) + if email_object: + email_object.add_reference(attribute.uuid, 'contains') + if self.config.sighting: + self.sighting(h, self.config.sighting_source) + for h in set(re.findall(hashmarker.SHA256_REGEX, body)): + contains_hash = True + attribute = self.misp_event.add_attribute('sha256', h, enforceWarninglist=self.config.enforcewarninglist) + if email_object: + email_object.add_reference(attribute.uuid, 'contains') + if self.config.sighting: + self.sighting(h, self.config.sighting_source) + + if contains_hash: + [self.misp_event.add_tag(tag) for tag in self.config.hash_only_tags] + + # # Extract network IOCs + urllist = [] + urllist += re.findall(urlmarker.WEB_URL_REGEX, body) + urllist += re.findall(urlmarker.IP_REGEX, body) + if self.debug: + syslog.syslog(str(urllist)) + + hostname_processed = [] + + # Add IOCs and expanded information to MISP + for entry in set(urllist): + ids_flag = True + self.f.decode(entry) + + domainname = self.f.get_domain().decode() + if domainname in self.config.excludelist: + # Ignore the entry + continue + + hostname = self.f.get_host().decode() + + scheme = self.f.get_scheme() + if scheme: + scheme = scheme.decode() + + resource_path = self.f.get_resource_path() + if resource_path: + resource_path = resource_path.decode() + + if self.debug: + syslog.syslog(domainname) + + if domainname in self.config.internallist: # Add link to internal reference + attribute = self.misp_event.add_attribute('link', entry, category='Internal reference', + to_ids=False, enforceWarninglist=False) + if email_object: + email_object.add_reference(attribute.uuid, 'contains') + elif domainname in self.config.externallist: # External analysis + attribute = self.misp_event.add_attribute('link', entry, category='External analysis', + to_ids=False, enforceWarninglist=False) + if email_object: + email_object.add_reference(attribute.uuid, 'contains') + else: # The URL is probably an indicator. + comment = "" + if (domainname in self.config.noidsflaglist) or (hostname in self.config.noidsflaglist): + ids_flag = False + comment = "Known host (mostly for connectivity test or IP lookup)" + if self.debug: + syslog.syslog(str(entry)) + + if scheme: + if is_ip(hostname): + attribute = self.misp_event.add_attribute('url', entry, to_ids=False, + enforceWarninglist=self.config.enforcewarninglist) + if email_object: + email_object.add_reference(attribute.uuid, 'contains') + else: + if resource_path: # URL has path, ignore warning list + attribute = self.misp_event.add_attribute('url', entry, to_ids=ids_flag, + enforceWarninglist=False, comment=comment) + if email_object: + email_object.add_reference(attribute.uuid, 'contains') + else: # URL has no path + attribute = self.misp_event.add_attribute('url', entry, to_ids=ids_flag, + enforceWarninglist=self.config.enforcewarninglist, comment=comment) + if email_object: + email_object.add_reference(attribute.uuid, 'contains') + if self.config.sighting: + self.sighting(entry, self.config.sighting_source) + + if hostname in hostname_processed: + # Hostname already processed. + continue + + hostname_processed.append(hostname) + if self.config.sighting: + self.sighting(hostname, self.config.sighting_source) + + if self.debug: + syslog.syslog(hostname) + + comment = '' + port = self.f.get_port() + if port: + port = port.decode() + comment = f'on port: {port}' + + if is_ip(hostname): + attribute = self.misp_event.add_attribute('ip-dst', hostname, to_ids=ids_flag, + enforceWarninglist=self.config.enforcewarninglist, + comment=comment) + if email_object: + email_object.add_reference(attribute.uuid, 'contains') + else: + related_ips = [] + try: + syslog.syslog(hostname) + for rdata in dns.resolver.query(hostname, 'A'): + if self.debug: + syslog.syslog(str(rdata)) + related_ips.append(rdata.to_text()) + except Exception as e: + if self.debug: + syslog.syslog(str(e)) + + if related_ips: + hip = MISPObject(name='ip-port') + hip.add_attribute('hostname', value=hostname, to_ids=ids_flag, + enforceWarninglist=self.config.enforcewarninglist, comment=comment) + for ip in set(related_ips): + hip.add_attribute('ip', type='ip-dst', value=ip, to_ids=False, + enforceWarninglist=self.config.enforcewarninglist) + self.misp_event.add_object(hip) + if email_object: + email_object.add_reference(hip.uuid, 'contains') + else: + attribute = self.misp_event.add_attribute('hostname', value=hostname, + to_ids=ids_flag, enforceWarninglist=self.config.enforcewarninglist, + comment=comment) + if email_object: + email_object.add_reference(attribute.uuid, 'contains') + + def add_event(self): + '''Add event on the remote MISP instance.''' + + # Add additional tags depending on others + tags = [] + for tag in [t.name for t in self.misp_event.tags]: + if self.config.dependingtags.get(tag): + tags += self.config.dependingtags.get(tag) + + # Add additional tags according to configuration + for malware in self.config.malwaretags: + if malware.lower() in self.subject.lower(): + tags += self.config.malwaretags.get(malware) + if tags: + [self.misp_event.add_tag(tag) for tag in tags] + + has_tlp_tag = False + for tag in [t.name for t in self.misp_event.tags]: + if tag.lower().startswith('tlp'): + has_tlp_tag = True + if not has_tlp_tag: + self.misp_event.add_tag(self.config.tlptag_default) + + if self.offline: + return self.misp_event.to_json() + return self.misp.add_event(self.misp_event) diff --git a/urlmarker.py b/mail2misp/urlmarker.py old mode 100755 new mode 100644 similarity index 100% rename from urlmarker.py rename to mail2misp/urlmarker.py diff --git a/mail_to_misp.py b/mail_to_misp.py index 04cce6e..8b70161 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -4,355 +4,12 @@ import os import sys import argparse -import re import syslog from pathlib import Path -import html from io import BytesIO -from ipaddress import ip_address -from email import message_from_bytes, policy import importlib -try: - import urlmarker - import hashmarker - from pyfaup.faup import Faup - from pymisp import PyMISP, MISPEvent, MISPObject, MISPSighting - from pymisp.tools import EMailObject, make_binary_objects - from defang import refang - import dns.resolver -except ImportError as e: - print("(!) Problem loading module:") - print(e) - sys.exit(-1) - - -def is_ip(address): - try: - ip_address(address) - except ValueError: - return False - return True - - -class Mail2MISP(): - - def __init__(self, misp_url, misp_key, verifycert, config): - self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug) - self.config = config - self.debug = self.config.debug - self.config_from_email_body = {} - # Init Faup - self.f = Faup() - - def load_email(self, pseudofile): - self.pseudofile = pseudofile - self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default) - self.subject = self.original_mail.get('Subject') - # Remove words from subject - for removeword in self.config.removelist: - self.subject = re.sub(removeword, "", self.subject).strip() - - # Initialize the MISP event - self.misp_event = MISPEvent() - self.misp_event.info = f'{config.email_subject_prefix} - {self.subject}' - self.misp_event.distribution = self.config.default_distribution - self.misp_event.threat_level_id = self.config.default_threat_level - self.misp_event.analysis = self.config.default_analysis - - def sighting(self, value, source): - '''Add a sighting''' - s = MISPSighting() - s.from_dict(value=value, source=source) - self.misp.set_sightings(s) - - def _find_inline_forward(self): - '''Does the body contains a forwarded email?''' - for identifier in self.config.forward_identifiers: - if identifier in self.clean_email_body: - self.clean_email_body, fw_email = self.clean_email_body.split(identifier) - return self.forwarded_email(pseudofile=BytesIO(fw_email.encode())) - - def _find_attached_forward(self): - forwarded_emails = [] - for attachment in self.original_mail.iter_attachments(): - # Search for email forwarded as attachment - # I could have more than one, attaching everything. - if attachment.get_filename() and attachment.get_filename().endswith('.eml'): - forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content().as_bytes()))) - else: - filename = attachment.get_filename() - if not filename: - filename = 'missing_filename' - if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword: - # Attach sane file - self.misp_event.add_attribute('attachment', value=filename, data=BytesIO(attachment.get_content())) - else: - f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content()), filename=filename, standalone=False) - self.misp_event.add_object(f_object) - if main_object: - self.misp_event.add_object(main_object) - [self.misp_event.add_object(section) for section in sections] - return forwarded_emails - - def email_from_spamtrap(self): - '''The email comes from a spamtrap and should be attached as-is.''' - raw_body = self.original_mail.get_body(preferencelist=('html', 'plain')) - if raw_body: - self.clean_email_body = html.unescape(raw_body.get_payload(decode=True).decode('utf8', 'surrogateescape')) - else: - self.clean_email_body = '' - return self.forwarded_email(self.pseudofile) - - def forwarded_email(self, pseudofile: BytesIO): - '''Extracts all possible indicators out of an email and create a MISP event out of it. - * Gets all relevant Headers - * Attach the body - * Create MISP file objects (uses lief if possible) - * Set all references - ''' - email_object = EMailObject(pseudofile=pseudofile, attach_original_mail=True, standalone=False) - if email_object.attachments: - # Create file objects for the attachments - for attachment_name, attachment in email_object.attachments: - if not attachment_name: - attachment_name = 'NameMissing.txt' - f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) - self.misp_event.add_object(f_object) - if main_object: - self.misp_event.add_object(main_object) - for section in sections: - self.misp_event.add_object(section) - email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment') - self.process_body_iocs(email_object) - if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get('attach_original_mail'): - self.misp_event.add_object(email_object) - return email_object - - def process_email_body(self): - mail_as_bytes = self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True) - if mail_as_bytes: - self.clean_email_body = html.unescape(mail_as_bytes.decode('utf8', 'surrogateescape')) - # Check if there are config lines in the body & convert them to a python dictionary: - # :: => {: } - self.config_from_email_body = {k.strip(): v.strip() for k, v in re.findall(f'{config.body_config_prefix}:(.*):(.*)', self.clean_email_body)} - if self.config_from_email_body: - # ... remove the config lines from the body - self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '', - html.unescape(self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape')), flags=re.MULTILINE) - - # Check if autopublish key is present and valid - if self.config_from_email_body.get('m2mkey') == self.config.m2m_key: - if self.config_from_email_body.get('distribution'): - self.misp_event.distribution = self.config_from_email_body.get('distribution') - if self.config_from_email_body.get('threat_level'): - self.misp_event.threat_level_id = self.config_from_email_body.get('threat_level') - if self.config_from_email_body.get('analysis'): - self.misp_event.analysis = self.config_from_email_body.get('analysis') - if self.config_from_email_body.get('publish'): - self.misp_event.publish() - - self._find_inline_forward() - else: - self.clean_email_body = '' - self._find_attached_forward() - - def process_body_iocs(self, email_object=None): - if email_object: - body = html.unescape(email_object.email.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape')) - else: - body = self.clean_email_body - - # Cleanup body content - # Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message - for ignoreline in self.config.ignorelist: - body = re.sub(rf'^{ignoreline}.*\n?', '', body, flags=re.MULTILINE) - - # Remove everything after the stopword from the body - body = body.split(self.config.stopword, 1)[0] - - # Add tags to the event if keywords are found in the mail - for tag in self.config.tlptags: - if any(alternativetag in body.lower() for alternativetag in self.config.tlptags[tag]): - self.misp_event.add_tag(tag) - - # Prepare extraction of IOCs - # Refang email data - body = refang(body) - - # Extract and add hashes - contains_hash = False - for h in set(re.findall(hashmarker.MD5_REGEX, body)): - contains_hash = True - attribute = self.misp_event.add_attribute('md5', h, enforceWarninglist=config.enforcewarninglist) - if email_object: - email_object.add_reference(attribute.uuid, 'contains') - if self.config.sighting: - self.sighting(h, self.config.sighting_source) - for h in set(re.findall(hashmarker.SHA1_REGEX, body)): - contains_hash = True - attribute = self.misp_event.add_attribute('sha1', h, enforceWarninglist=config.enforcewarninglist) - if email_object: - email_object.add_reference(attribute.uuid, 'contains') - if self.config.sighting: - self.sighting(h, self.config.sighting_source) - for h in set(re.findall(hashmarker.SHA256_REGEX, body)): - contains_hash = True - attribute = self.misp_event.add_attribute('sha256', h, enforceWarninglist=config.enforcewarninglist) - if email_object: - email_object.add_reference(attribute.uuid, 'contains') - if self.config.sighting: - self.sighting(h, self.config.sighting_source) - - if contains_hash: - [self.misp_event.add_tag(tag) for tag in self.config.hash_only_tags] - - # # Extract network IOCs - urllist = [] - urllist += re.findall(urlmarker.WEB_URL_REGEX, body) - urllist += re.findall(urlmarker.IP_REGEX, body) - if self.debug: - syslog.syslog(str(urllist)) - - hostname_processed = [] - - # Add IOCs and expanded information to MISP - for entry in set(urllist): - ids_flag = True - self.f.decode(entry) - - domainname = self.f.get_domain().decode() - if domainname in self.config.excludelist: - # Ignore the entry - continue - - hostname = self.f.get_host().decode() - - scheme = self.f.get_scheme() - if scheme: - scheme = scheme.decode() - - resource_path = self.f.get_resource_path() - if resource_path: - resource_path = resource_path.decode() - - if debug: - syslog.syslog(domainname) - - if domainname in self.config.internallist: # Add link to internal reference - attribute = self.misp_event.add_attribute('link', entry, category='Internal reference', - to_ids=False, enforceWarninglist=False) - if email_object: - email_object.add_reference(attribute.uuid, 'contains') - elif domainname in self.config.externallist: # External analysis - attribute = self.misp_event.add_attribute('link', entry, category='External analysis', - to_ids=False, enforceWarninglist=False) - if email_object: - email_object.add_reference(attribute.uuid, 'contains') - else: # The URL is probably an indicator. - comment = "" - if (domainname in self.config.noidsflaglist) or (hostname in self.config.noidsflaglist): - ids_flag = False - comment = "Known host (mostly for connectivity test or IP lookup)" - if debug: - syslog.syslog(str(entry)) - - if scheme: - if is_ip(hostname): - attribute = self.misp_event.add_attribute('url', entry, to_ids=False, - enforceWarninglist=config.enforcewarninglist) - if email_object: - email_object.add_reference(attribute.uuid, 'contains') - else: - if resource_path: # URL has path, ignore warning list - attribute = self.misp_event.add_attribute('url', entry, to_ids=ids_flag, - enforceWarninglist=False, comment=comment) - if email_object: - email_object.add_reference(attribute.uuid, 'contains') - else: # URL has no path - attribute = self.misp_event.add_attribute('url', entry, to_ids=ids_flag, - enforceWarninglist=config.enforcewarninglist, comment=comment) - if email_object: - email_object.add_reference(attribute.uuid, 'contains') - if self.config.sighting: - self.sighting(entry, self.config.sighting_source) - - if hostname in hostname_processed: - # Hostname already processed. - continue - - hostname_processed.append(hostname) - if self.config.sighting: - self.sighting(hostname, self.config.sighting_source) - - if debug: - syslog.syslog(hostname) - - comment = '' - port = self.f.get_port() - if port: - port = port.decode() - comment = f'on port: {port}' - - if is_ip(hostname): - attribute = self.misp_event.add_attribute('ip-dst', hostname, to_ids=ids_flag, - enforceWarninglist=config.enforcewarninglist, - comment=comment) - if email_object: - email_object.add_reference(attribute.uuid, 'contains') - else: - related_ips = [] - try: - syslog.syslog(hostname) - for rdata in dns.resolver.query(hostname, 'A'): - if debug: - syslog.syslog(str(rdata)) - related_ips.append(rdata.to_text()) - except Exception as e: - if debug: - syslog.syslog(str(e)) - - if related_ips: - hip = MISPObject(name='ip-port') - hip.add_attribute('hostname', value=hostname, to_ids=ids_flag, - enforceWarninglist=config.enforcewarninglist, comment=comment) - for ip in set(related_ips): - hip.add_attribute('ip', type='ip-dst', value=ip, to_ids=False, - enforceWarninglist=config.enforcewarninglist) - self.misp_event.add_object(hip) - if email_object: - email_object.add_reference(hip.uuid, 'contains') - else: - attribute = self.misp_event.add_attribute('hostname', value=hostname, - to_ids=ids_flag, enforceWarninglist=config.enforcewarninglist, - comment=comment) - if email_object: - email_object.add_reference(attribute.uuid, 'contains') - - def add_event(self): - '''Add event on the remote MISP instance.''' - - # Add additional tags depending on others - tags = [] - for tag in [t.name for t in self.misp_event.tags]: - if self.config.dependingtags.get(tag): - tags += self.config.dependingtags.get(tag) - - # Add additional tags according to configuration - for malware in self.config.malwaretags: - if malware.lower() in self.subject.lower(): - tags += self.config.malwaretags.get(malware) - if tags: - [self.misp_event.add_tag(tag) for tag in tags] - - has_tlp_tag = False - for tag in [t.name for t in self.misp_event.tags]: - if tag.lower().startswith('tlp'): - has_tlp_tag = True - if not has_tlp_tag: - self.misp_event.add_tag(config.tlptag_default) - - self.misp.add_event(self.misp_event) +from mail2misp import Mail2MISP if __name__ == '__main__': parser = argparse.ArgumentParser(description='Push a Mail into a MISP instance') diff --git a/mail_to_misp_config.py-example b/mail_to_misp_config.py-example index 34fb226..e170b0e 100644 --- a/mail_to_misp_config.py-example +++ b/mail_to_misp_config.py-example @@ -43,7 +43,7 @@ sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER" # Remove "Re:", "Fwd:" and {Spam?} from subject # add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag] -removelist = ("Re:", "Fwd:", "\{Spam\?\} ") +removelist = (r"Re:", r"Fwd:", r"\{Spam\?\} ") # TLP tag setup # Tuples contain different variations of spelling diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/config_forward.py b/tests/config_forward.py new file mode 100644 index 0000000..f988989 --- /dev/null +++ b/tests/config_forward.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +misp_url = 'YOUR_MISP_URL' +misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section +misp_verifycert = True +spamtrap = False +default_distribution = 0 +default_threat_level = 3 +default_analysis = 1 + +body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs +m2m_key = 'YOUSETYOURKEYHERE' +m2m_benign_attachment_keyword = 'benign' + +debug = True +nameservers = ['8.8.8.8'] +email_subject_prefix = 'M2M' +attach_original_mail = True + +excludelist = ('google.com', 'microsoft.com') +externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net') +internallist = ('internal.system.local') +noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net', + 'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com', + 'dropbox.com' + ) + +# Stop parsing when this term is found +stopword = 'Whois & IP Information' + +# Ignore lines in body of message containing: +ignorelist = ("From:", "Sender:", "Received:", "Sender IP:", "Reply-To:", "Registrar WHOIS Server:", + "Registrar:", "Domain Status:", "Registrant Email:", "IP Location:", + "X-Get-Message-Sender-Via:", "X-Authenticated-Sender:") + +# Ignore (don't add) attributes that are on server side warning list +enforcewarninglist = True + +# Add a sighting for each value +sighting = False +sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER" + +# Remove "Re:", "Fwd:" and {Spam?} from subject +# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag] +removelist = (r'Re:', r'Fwd:', r'\{Spam?\}') + +# TLP tag setup +# Tuples contain different variations of spelling +tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'], + 'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'], + 'tlp:white': ['tlp:white', 'tlp: white', 'tlp white'] + } +tlptag_default = sorted(tlptags.keys())[0] + +malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"'], + 'jaff': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"'], + 'dridex': ['misp-galaxy:tool="dridex"'], + 'netwire': ['Netwire RAT'], + 'Pony': ['misp-galaxy:tool="Hancitor"'], + 'ursnif': ['misp-galaxy:tool="Snifula"'], + 'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'], + 'trickbot': ['misp-galaxy:tool="Trick Bot"'] + } + +# Tags to be set depending on the presence of other tags +dependingtags = {'tlp:white': ['circl:osint-feed'] + } + +# Known identifiers for forwarded messages +forward_identifiers = {'-------- Forwarded Message --------', 'Begin forwarded message:'} + +# Tags to add when hashes are found (e.g. to do automatic expansion) +hash_only_tags = {'TODO:VT-ENRICHMENT'} + +# If an attribute is on any MISP server side `warning list`, skip the creation of the attribute +skip_item_on_warninglist = True diff --git a/tests/config_spamtrap.py b/tests/config_spamtrap.py new file mode 100644 index 0000000..8f5dad4 --- /dev/null +++ b/tests/config_spamtrap.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +misp_url = 'YOUR_MISP_URL' +misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section +misp_verifycert = True +spamtrap = True +default_distribution = 0 +default_threat_level = 3 +default_analysis = 1 + +body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs +m2m_key = 'YOUSETYOURKEYHERE' +m2m_benign_attachment_keyword = 'benign' + +debug = True +nameservers = ['8.8.8.8'] +email_subject_prefix = 'M2M' +attach_original_mail = True + +excludelist = ('google.com', 'microsoft.com') +externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net') +internallist = ('internal.system.local') +noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net', + 'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com', + 'dropbox.com' + ) + +# Stop parsing when this term is found +stopword = 'Whois & IP Information' + +# Ignore lines in body of message containing: +ignorelist = ("From:", "Sender:", "Received:", "Sender IP:", "Reply-To:", "Registrar WHOIS Server:", + "Registrar:", "Domain Status:", "Registrant Email:", "IP Location:", + "X-Get-Message-Sender-Via:", "X-Authenticated-Sender:") + +# Ignore (don't add) attributes that are on server side warning list +enforcewarninglist = True + +# Add a sighting for each value +sighting = False +sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER" + +# Remove "Re:", "Fwd:" and {Spam?} from subject +# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag] +removelist = (r'Re:', r'Fwd:', r'\{Spam\?\}') + +# TLP tag setup +# Tuples contain different variations of spelling +tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'], + 'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'], + 'tlp:white': ['tlp:white', 'tlp: white', 'tlp white'] + } +tlptag_default = sorted(tlptags.keys())[0] + +malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"'], + 'jaff': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"'], + 'dridex': ['misp-galaxy:tool="dridex"'], + 'netwire': ['Netwire RAT'], + 'Pony': ['misp-galaxy:tool="Hancitor"'], + 'ursnif': ['misp-galaxy:tool="Snifula"'], + 'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'], + 'trickbot': ['misp-galaxy:tool="Trick Bot"'] + } + +# Tags to be set depending on the presence of other tags +dependingtags = {'tlp:white': ['circl:osint-feed'] + } + +# Known identifiers for forwarded messages +forward_identifiers = {'-------- Forwarded Message --------', 'Begin forwarded message:'} + +# Tags to add when hashes are found (e.g. to do automatic expansion) +hash_only_tags = {'TODO:VT-ENRICHMENT'} + +# If an attribute is on any MISP server side `warning list`, skip the creation of the attribute +skip_item_on_warninglist = True diff --git a/tests/mails b/tests/mails new file mode 160000 index 0000000..1e673f9 --- /dev/null +++ b/tests/mails @@ -0,0 +1 @@ +Subproject commit 1e673f9d64e24136a9e4ef3f731c1a6052f9bb87 diff --git a/tests/tests.py b/tests/tests.py new file mode 100644 index 0000000..8bb8952 --- /dev/null +++ b/tests/tests.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +import importlib +import sys +from io import BytesIO +sys.path.insert(0, ".") + +from mail2misp import Mail2MISP + + +class TestMailToMISP(unittest.TestCase): + + def test_spamtrap(self): + config = importlib.import_module('tests.config_spamtrap') + self.mail2misp = Mail2MISP('', '', '', config=config, offline=True) + with open('tests/mails/simple_spamtrap.eml', 'rb') as f: + self.mail2misp.load_email(BytesIO(f.read())) + self.mail2misp.email_from_spamtrap() + self.mail2misp.process_body_iocs() + event = self.mail2misp.add_event() + print(event) + + def test_spamtrap_attachment(self): + config = importlib.import_module('tests.config_spamtrap') + self.mail2misp = Mail2MISP('', '', '', config=config, offline=True) + with open('tests/mails/attachment_spamtrap.eml', 'rb') as f: + self.mail2misp.load_email(BytesIO(f.read())) + self.mail2misp.email_from_spamtrap() + self.mail2misp.process_body_iocs() + event = self.mail2misp.add_event() + print(event) + + def test_forward(self): + config = importlib.import_module('tests.config_forward') + self.mail2misp = Mail2MISP('', '', '', config=config, offline=True) + with open('tests/mails/simple_forward.eml', 'rb') as f: + self.mail2misp.load_email(BytesIO(f.read())) + self.mail2misp.process_email_body() + self.mail2misp.process_body_iocs() + event = self.mail2misp.add_event() + print(event) + + def test_forward_attachment(self): + config = importlib.import_module('tests.config_forward') + self.mail2misp = Mail2MISP('', '', '', config=config, offline=True) + with open('tests/mails/attachment_forward.eml', 'rb') as f: + self.mail2misp.load_email(BytesIO(f.read())) + self.mail2misp.process_email_body() + self.mail2misp.process_body_iocs() + event = self.mail2misp.add_event() + print(event) + + +if __name__ == '__main__': + unittest.main() From 6d829713e4580a034b1bcec5735df2e12a88f7af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 14 May 2018 23:13:13 -0400 Subject: [PATCH 25/50] new: Add badges --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ba1f2a3..42838b4 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +[![Build Status](https://travis-ci.org/MISP/mail_to_misp.svg?branch=master)](https://travis-ci.org/MISP/mail_to_misp) +[![codecov](https://codecov.io/gh/MISP/mail_to_misp/branch/master/graph/badge.svg)](https://codecov.io/gh/MISP/mail_to_misp) + # mail_to_misp Connect your mail infrastructure to [MISP](https://github.com/MISP/MISP) in order to create events based on the information contained within mails. From 7a1b3c0189e0028f69c3346bd0a9161aebcbb6fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 14 May 2018 23:16:01 -0400 Subject: [PATCH 26/50] fix: missing deps for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b3a5921..d69efc5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ install: - sudo make install - popd - sudo ldconfig - - pip install -U nose pip coverage + - pip install -U nose pip coverage codecov coveralls - pip install -U -r requirements.txt - git submodule init - git submodule update From 64180ecc14bf839bdd0b1057c73f85fc0b6ebc35 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Tue, 12 Jun 2018 23:28:16 +0900 Subject: [PATCH 27/50] - Added mail_to_misp_test submodule --- .gitmodules | 3 +++ mail_to_misp_test | 1 + 2 files changed, 4 insertions(+) create mode 160000 mail_to_misp_test diff --git a/.gitmodules b/.gitmodules index 1044271..64dc30a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "tests/mails"] path = tests/mails url = https://github.com/MISP/mail_to_misp_test.git +[submodule "mail_to_misp_test"] + path = mail_to_misp_test + url = git@github.com:MISP/mail_to_misp_test.git diff --git a/mail_to_misp_test b/mail_to_misp_test new file mode 160000 index 0000000..1e673f9 --- /dev/null +++ b/mail_to_misp_test @@ -0,0 +1 @@ +Subproject commit 1e673f9d64e24136a9e4ef3f731c1a6052f9bb87 From 5ec8a2fa9082e7bd99336708147420160bd365d3 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Tue, 12 Jun 2018 23:50:34 +0900 Subject: [PATCH 28/50] =?UTF-8?q?-=20Remove=20submodule=20again=20as=20I?= =?UTF-8?q?=20noticed=20it=20was=20already=20present,=20sorry=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 64dc30a..1044271 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "tests/mails"] path = tests/mails url = https://github.com/MISP/mail_to_misp_test.git -[submodule "mail_to_misp_test"] - path = mail_to_misp_test - url = git@github.com:MISP/mail_to_misp_test.git From c823e5496c18642264175ac92dbca26b965598b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Aug 2018 11:51:11 +0200 Subject: [PATCH 29/50] fix: benign attachment in FW email Fix #25 --- mail2misp/mail2misp.py | 18 +++++++++++------- tests/tests.py | 8 ++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py index d2218d8..9d651d8 100644 --- a/mail2misp/mail2misp.py +++ b/mail2misp/mail2misp.py @@ -111,13 +111,17 @@ class Mail2MISP(): for attachment_name, attachment in email_object.attachments: if not attachment_name: attachment_name = 'NameMissing.txt' - f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) - self.misp_event.add_object(f_object) - if main_object: - self.misp_event.add_object(main_object) - for section in sections: - self.misp_event.add_object(section) - email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment') + if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword: + a = self.misp_event.add_attribute('attachment', value=attachment_name, data=attachment) + email_object.add_reference(a.uuid, 'related-to', 'Email attachment') + else: + f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) + self.misp_event.add_object(f_object) + if main_object: + self.misp_event.add_object(main_object) + for section in sections: + self.misp_event.add_object(section) + email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment') self.process_body_iocs(email_object) if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get('attach_original_mail'): self.misp_event.add_object(email_object) diff --git a/tests/tests.py b/tests/tests.py index 8bb8952..97ff3e6 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -52,6 +52,14 @@ class TestMailToMISP(unittest.TestCase): event = self.mail2misp.add_event() print(event) + def test_benign(self): + config = importlib.import_module('tests.config_forward') + self.mail2misp = Mail2MISP('', '', '', config=config, offline=True) + with open('tests/mails/test_benign.eml', 'rb') as f: + self.mail2misp.load_email(BytesIO(f.read())) + self.mail2misp.process_email_body() + self.mail2misp.process_body_iocs() + self.assertTrue('attachment' in [a.type for a in self.mail2misp.misp_event.attributes]) if __name__ == '__main__': unittest.main() From cbf0c07b674a2a21a5e3ff5304241e41f80cbfa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Aug 2018 11:55:37 +0200 Subject: [PATCH 30/50] new: Allow to disable DNS lookups Fix #26 --- mail2misp/mail2misp.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py index 9d651d8..e569297 100644 --- a/mail2misp/mail2misp.py +++ b/mail2misp/mail2misp.py @@ -14,7 +14,11 @@ from pyfaup.faup import Faup from pymisp import PyMISP, MISPEvent, MISPObject, MISPSighting from pymisp.tools import EMailObject, make_binary_objects from defang import refang -import dns.resolver +try: + import dns.resolver + HAS_DNS = True +except ImportError: + HAS_DNS = False def is_ip(address): @@ -299,15 +303,16 @@ class Mail2MISP(): email_object.add_reference(attribute.uuid, 'contains') else: related_ips = [] - try: - syslog.syslog(hostname) - for rdata in dns.resolver.query(hostname, 'A'): + if HAS_DNS and self.config.enable_dns: + try: + syslog.syslog(hostname) + for rdata in dns.resolver.query(hostname, 'A'): + if self.debug: + syslog.syslog(str(rdata)) + related_ips.append(rdata.to_text()) + except Exception as e: if self.debug: - syslog.syslog(str(rdata)) - related_ips.append(rdata.to_text()) - except Exception as e: - if self.debug: - syslog.syslog(str(e)) + syslog.syslog(str(e)) if related_ips: hip = MISPObject(name='ip-port') From 02b1c8ed96c207007945de8c8fefa0dfad146c0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Aug 2018 12:02:18 +0200 Subject: [PATCH 31/50] fix: Avoid failure if dns key is not in the config file. --- mail2misp/mail2misp.py | 2 ++ tests/tests.py | 1 + 2 files changed, 3 insertions(+) diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py index e569297..0c984b9 100644 --- a/mail2misp/mail2misp.py +++ b/mail2misp/mail2misp.py @@ -36,6 +36,8 @@ class Mail2MISP(): if not self.offline: self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug) self.config = config + if not hasattr(self.config, 'enable_dns'): + setattr(self.config, 'enable_dns', True) self.debug = self.config.debug self.config_from_email_body = {} # Init Faup diff --git a/tests/tests.py b/tests/tests.py index 97ff3e6..27cfcda 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -60,6 +60,7 @@ class TestMailToMISP(unittest.TestCase): self.mail2misp.process_email_body() self.mail2misp.process_body_iocs() self.assertTrue('attachment' in [a.type for a in self.mail2misp.misp_event.attributes]) + self.assertTrue(self.mail2misp.misp_event.publish) if __name__ == '__main__': unittest.main() From cdfe86af2a9fc46435d576cc06470ee99abcdc2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Aug 2018 12:02:50 +0200 Subject: [PATCH 32/50] chg: Update readme --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 42838b4..d13ee0a 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,22 @@ If you send a mail to mail_to_misp containing: `key:ABCDEFGHIJKLMN0PQRSTUVWXYZ` If you don't want to use this feature, just don't put it in the message body. The distribution is defined in the configuration as well: `m2m_auto_distribution = '3' # 3 = All communities` +# Pass parameters in the email boddy + +``` +m2m:: + +# Examples +m2m:attachment:benign # Email attachment considered benign (attachment in MISP, malware-sample by default) +m2m:attach_original_mail:1 # Attach the full original email to the MISP Event (may contain private information) + +m2m:m2mkey:YOUSETYOURKEYHERE # Key required for some actions +# The following key are ignored if M2M:m2mkey is invalid +m2m:distribution:<0-3,5> # Note: impossible to pass a sharing group yet. +m2m:threat_level:<0-2> +m2m:analysis:<0-3> +m2m:publish:1 # Autopublish +``` ## Implementation From 03bf3e3506af85f522ad7ce714a5ba66ab1e89ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Aug 2018 13:42:07 +0200 Subject: [PATCH 33/50] fix: Proper type detection of attachments Fix #27 --- mail2misp/mail2misp.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py index 0c984b9..e33109b 100644 --- a/mail2misp/mail2misp.py +++ b/mail2misp/mail2misp.py @@ -6,7 +6,7 @@ import syslog import html from io import BytesIO from ipaddress import ip_address -from email import message_from_bytes, policy +from email import message_from_bytes, policy, message from . import urlmarker from . import hashmarker @@ -76,19 +76,22 @@ class Mail2MISP(): def _find_attached_forward(self): forwarded_emails = [] for attachment in self.original_mail.iter_attachments(): + attachment_content = attachment.get_content() # Search for email forwarded as attachment # I could have more than one, attaching everything. - if attachment.get_filename() and attachment.get_filename().endswith('.eml'): - forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content().as_bytes()))) + if isinstance(attachment_content, message.EmailMessage): + forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment_content.as_bytes()))) else: + if isinstance(attachment_content, str): + attachment_content = BytesIO(attachment_content.encode()) filename = attachment.get_filename() if not filename: filename = 'missing_filename' if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword: # Attach sane file - self.misp_event.add_attribute('attachment', value=filename, data=BytesIO(attachment.get_content())) + self.misp_event.add_attribute('attachment', value=filename, data=BytesIO(attachment_content)) else: - f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content()), filename=filename, standalone=False) + f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment_content), filename=filename, standalone=False) self.misp_event.add_object(f_object) if main_object: self.misp_event.add_object(main_object) From 59f630268b6d1feac232a6e4c71215ddca28041e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Aug 2018 13:44:53 +0200 Subject: [PATCH 34/50] fix: Add submodule --- tests/mails | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mails b/tests/mails index 1e673f9..64c0ff1 160000 --- a/tests/mails +++ b/tests/mails @@ -1 +1 @@ -Subproject commit 1e673f9d64e24136a9e4ef3f731c1a6052f9bb87 +Subproject commit 64c0ff12d12b98c59b49b961949d7f0f4a95219a From 086ab9f367d298f0b21cc4ce2ed0ffb948cb6f2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Aug 2018 13:51:11 +0200 Subject: [PATCH 35/50] fix: Fucked up submodule --- mail_to_misp_test | 1 - tests/mails | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 160000 mail_to_misp_test diff --git a/mail_to_misp_test b/mail_to_misp_test deleted file mode 160000 index 1e673f9..0000000 --- a/mail_to_misp_test +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1e673f9d64e24136a9e4ef3f731c1a6052f9bb87 diff --git a/tests/mails b/tests/mails index 64c0ff1..1e673f9 160000 --- a/tests/mails +++ b/tests/mails @@ -1 +1 @@ -Subproject commit 64c0ff12d12b98c59b49b961949d7f0f4a95219a +Subproject commit 1e673f9d64e24136a9e4ef3f731c1a6052f9bb87 From 517bb19786874ff0b3f4cf31e649a5318b10df17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Aug 2018 13:57:21 +0200 Subject: [PATCH 36/50] chg: Dump test files --- tests/mails | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mails b/tests/mails index 1e673f9..d57c26e 160000 --- a/tests/mails +++ b/tests/mails @@ -1 +1 @@ -Subproject commit 1e673f9d64e24136a9e4ef3f731c1a6052f9bb87 +Subproject commit d57c26e25f6da499a806de2b60f1861f3ed882af From 5facd8d4d5884cf9133ea61fa12c6510285a9ab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Aug 2018 14:02:10 +0200 Subject: [PATCH 37/50] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d13ee0a..b4a7dfb 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ If you send a mail to mail_to_misp containing: `key:ABCDEFGHIJKLMN0PQRSTUVWXYZ` If you don't want to use this feature, just don't put it in the message body. The distribution is defined in the configuration as well: `m2m_auto_distribution = '3' # 3 = All communities` -# Pass parameters in the email boddy +# Pass parameters in the email body ``` m2m:: From 45fa0c7323bf1f77cd9f8f4c6fa9d020d156932b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Aug 2018 17:02:17 +0200 Subject: [PATCH 38/50] fix: Properly handle plain text attachments --- mail2misp/mail2misp.py | 2 +- tests/mails | 2 +- tests/tests.py | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py index e33109b..5c5d613 100644 --- a/mail2misp/mail2misp.py +++ b/mail2misp/mail2misp.py @@ -83,7 +83,7 @@ class Mail2MISP(): forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment_content.as_bytes()))) else: if isinstance(attachment_content, str): - attachment_content = BytesIO(attachment_content.encode()) + attachment_content = attachment_content.encode() filename = attachment.get_filename() if not filename: filename = 'missing_filename' diff --git a/tests/mails b/tests/mails index d57c26e..58d3cfb 160000 --- a/tests/mails +++ b/tests/mails @@ -1 +1 @@ -Subproject commit d57c26e25f6da499a806de2b60f1861f3ed882af +Subproject commit 58d3cfbe1355aa184e0d68fcda88ead371974574 diff --git a/tests/tests.py b/tests/tests.py index 27cfcda..b80443f 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -62,5 +62,13 @@ class TestMailToMISP(unittest.TestCase): self.assertTrue('attachment' in [a.type for a in self.mail2misp.misp_event.attributes]) self.assertTrue(self.mail2misp.misp_event.publish) + def test_textfile(self): + config = importlib.import_module('tests.config_forward') + self.mail2misp = Mail2MISP('', '', '', config=config, offline=True) + with open('tests/mails/test_textattachment.eml', 'rb') as f: + self.mail2misp.load_email(BytesIO(f.read())) + self.mail2misp.process_email_body() + + if __name__ == '__main__': unittest.main() From 145930d55d2b22b04ff64634613f921e931dcecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Aug 2018 17:04:46 +0200 Subject: [PATCH 39/50] new: Add config to disable DNS lookups --- mail_to_misp_config.py-example | 1 + 1 file changed, 1 insertion(+) diff --git a/mail_to_misp_config.py-example b/mail_to_misp_config.py-example index e170b0e..3c7096b 100644 --- a/mail_to_misp_config.py-example +++ b/mail_to_misp_config.py-example @@ -13,6 +13,7 @@ body_config_prefix = 'm2m' # every line in the body starting with this value wi m2m_key = 'YOUSETYOURKEYHERE' m2m_benign_attachment_keyword = 'benign' +enable_dns = True debug = False nameservers = ['149.13.33.69'] email_subject_prefix = 'M2M' From 757f2cb4bf573ab3e90b48b88246240ed9f13de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 3 Aug 2018 10:52:35 +0200 Subject: [PATCH 40/50] fix: Allow passing 0 to distribution, threat_level and analysis --- mail2misp/mail2misp.py | 6 +++--- tests/tests.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py index 5c5d613..7573d00 100644 --- a/mail2misp/mail2misp.py +++ b/mail2misp/mail2misp.py @@ -149,11 +149,11 @@ class Mail2MISP(): html.unescape(self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape')), flags=re.MULTILINE) # Check if autopublish key is present and valid if self.config_from_email_body.get('m2mkey') == self.config.m2m_key: - if self.config_from_email_body.get('distribution'): + if self.config_from_email_body.get('distribution') is not None: self.misp_event.distribution = self.config_from_email_body.get('distribution') - if self.config_from_email_body.get('threat_level'): + if self.config_from_email_body.get('threat_level') is not None: self.misp_event.threat_level_id = self.config_from_email_body.get('threat_level') - if self.config_from_email_body.get('analysis'): + if self.config_from_email_body.get('analysis') is not None: self.misp_event.analysis = self.config_from_email_body.get('analysis') if self.config_from_email_body.get('publish'): self.misp_event.publish() diff --git a/tests/tests.py b/tests/tests.py index b80443f..b5cd949 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -69,6 +69,17 @@ class TestMailToMISP(unittest.TestCase): self.mail2misp.load_email(BytesIO(f.read())) self.mail2misp.process_email_body() + def test_meta_event(self): + config = importlib.import_module('tests.config_forward') + self.mail2misp = Mail2MISP('', '', '', config=config, offline=True) + with open('tests/mails/test_meta.eml', 'rb') as f: + self.mail2misp.load_email(BytesIO(f.read())) + self.mail2misp.process_email_body() + self.mail2misp.process_body_iocs() + self.assertTrue(self.mail2misp.misp_event.publish) + self.assertEqual(self.mail2misp.misp_event.distribution, 3) + self.assertEqual(self.mail2misp.misp_event.threat_level_id, 2) + self.assertEqual(self.mail2misp.misp_event.analysis, 0) if __name__ == '__main__': unittest.main() From 11c99c879bfe2f38e56e66922ca022226b7edc8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 3 Aug 2018 11:26:11 +0200 Subject: [PATCH 41/50] fix: Properly add sightings, meta event attributes --- mail2misp/mail2misp.py | 17 +++++++++++------ tests/tests.py | 8 +++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py index 7573d00..aa1709f 100644 --- a/mail2misp/mail2misp.py +++ b/mail2misp/mail2misp.py @@ -42,6 +42,7 @@ class Mail2MISP(): self.config_from_email_body = {} # Init Faup self.f = Faup() + self.sightings_to_add = [] def load_email(self, pseudofile): self.pseudofile = pseudofile @@ -194,21 +195,21 @@ class Mail2MISP(): if email_object: email_object.add_reference(attribute.uuid, 'contains') if self.config.sighting: - self.sighting(h, self.config.sighting_source) + self.sightings_to_add.append((h, self.config.sighting_source)) for h in set(re.findall(hashmarker.SHA1_REGEX, body)): contains_hash = True attribute = self.misp_event.add_attribute('sha1', h, enforceWarninglist=self.config.enforcewarninglist) if email_object: email_object.add_reference(attribute.uuid, 'contains') if self.config.sighting: - self.sighting(h, self.config.sighting_source) + self.sightings_to_add.append((h, self.config.sighting_source)) for h in set(re.findall(hashmarker.SHA256_REGEX, body)): contains_hash = True attribute = self.misp_event.add_attribute('sha256', h, enforceWarninglist=self.config.enforcewarninglist) if email_object: email_object.add_reference(attribute.uuid, 'contains') if self.config.sighting: - self.sighting(h, self.config.sighting_source) + self.sightings_to_add.append((h, self.config.sighting_source)) if contains_hash: [self.misp_event.add_tag(tag) for tag in self.config.hash_only_tags] @@ -281,7 +282,7 @@ class Mail2MISP(): if email_object: email_object.add_reference(attribute.uuid, 'contains') if self.config.sighting: - self.sighting(entry, self.config.sighting_source) + self.sightings_to_add.append((entry, self.config.sighting_source)) if hostname in hostname_processed: # Hostname already processed. @@ -289,7 +290,7 @@ class Mail2MISP(): hostname_processed.append(hostname) if self.config.sighting: - self.sighting(hostname, self.config.sighting_source) + self.sightings_to_add.append((hostname, self.config.sighting_source)) if self.debug: syslog.syslog(hostname) @@ -361,4 +362,8 @@ class Mail2MISP(): if self.offline: return self.misp_event.to_json() - return self.misp.add_event(self.misp_event) + event = self.misp.add_event(self.misp_event) + if self.config.sighting: + for value, source in self.sightings_to_add: + self.sighting(value, source) + return event diff --git a/tests/tests.py b/tests/tests.py index b5cd949..42bde92 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -77,9 +77,11 @@ class TestMailToMISP(unittest.TestCase): self.mail2misp.process_email_body() self.mail2misp.process_body_iocs() self.assertTrue(self.mail2misp.misp_event.publish) - self.assertEqual(self.mail2misp.misp_event.distribution, 3) - self.assertEqual(self.mail2misp.misp_event.threat_level_id, 2) - self.assertEqual(self.mail2misp.misp_event.analysis, 0) + self.assertEqual(self.mail2misp.misp_event.distribution, '3') + self.assertEqual(self.mail2misp.misp_event.threat_level_id, '2') + self.assertEqual(self.mail2misp.misp_event.analysis, '0') + self.mail2misp.add_event() + if __name__ == '__main__': unittest.main() From 7ba873dd68a42e7fcb2225cf6026aba1cf56fcfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 3 Aug 2018 11:48:57 +0200 Subject: [PATCH 42/50] chg: Bump test files --- tests/mails | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mails b/tests/mails index 58d3cfb..a20daf7 160000 --- a/tests/mails +++ b/tests/mails @@ -1 +1 @@ -Subproject commit 58d3cfbe1355aa184e0d68fcda88ead371974574 +Subproject commit a20daf72071403b01bdf3816d6de533b0a4d2c40 From 28326e37a984a5078d0b2416aaaa354686642a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 21 Jan 2019 14:33:09 +0100 Subject: [PATCH 43/50] chg: Use pipenv --- .travis.yml | 11 +-- Pipfile | 22 +++++ Pipfile.lock | 222 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 24 +++-- 4 files changed, 266 insertions(+), 13 deletions(-) create mode 100644 Pipfile create mode 100644 Pipfile.lock diff --git a/.travis.yml b/.travis.yml index d69efc5..221e772 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ addons: python: - "3.6" - "3.6-dev" + - "3.7-dev" install: - git clone git://github.com/stricaud/faup.git @@ -22,14 +23,14 @@ install: - sudo make install - popd - sudo ldconfig - - pip install -U nose pip coverage codecov coveralls - - pip install -U -r requirements.txt + - pip install pipenv + - pipenv install -d - git submodule init - git submodule update script: - - nosetests --with-coverage --cover-package=mail2misp tests/tests.py + - pipenv run nosetests --with-coverage --cover-package=mail2misp tests/tests.py after_success: - - codecov - - coveralls + - pipenv run codecov + - pipenv run coveralls diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..8d561ac --- /dev/null +++ b/Pipfile @@ -0,0 +1,22 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +nose = "*" +coverage = "*" +codecov = "*" +coveralls = "*" + +[packages] +dnspython = "*" +lief = "*" +python-magic = "*" +pydeep = {git = "https://github.com/kbandla/pydeep.git"} +pyfaup = {git = "https://github.com/stricaud/faup.git",subdirectory = "src/lib/bindings/python"} +defang = {git = "https://github.com/Rafiot/defang.git"} +pymisp = {editable = true,git = "https://github.com/MISP/PyMISP.git"} + +[requires] +python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..c1ee946 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,222 @@ +{ + "_meta": { + "hash": { + "sha256": "20759a97e7bb6bc062e147a56426b3039344319c4140e6312f3a3715b6265ad7" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + ], + "version": "==2018.11.29" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "defang": { + "git": "https://github.com/Rafiot/defang.git", + "ref": "52972a25313e2899f98f1777b940cb2122566a26" + }, + "dnspython": { + "hashes": [ + "sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01", + "sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d" + ], + "index": "pypi", + "version": "==1.16.0" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "jsonschema": { + "hashes": [ + "sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08", + "sha256:6ff5f3180870836cae40f06fa10419f557208175f13ad7bc26caa77beb1f6e02" + ], + "version": "==2.6.0" + }, + "lief": { + "hashes": [ + "sha256:c95974006a6b8a767eea8b35e6c63e2b20939730063ac472894b53ab9855a0b5" + ], + "index": "pypi", + "version": "==0.9.0" + }, + "pydeep": { + "git": "https://github.com/kbandla/pydeep.git", + "ref": "bc0d33bff4b45718b4c5f2c79d4715d92a427eda" + }, + "pyfaup": { + "git": "https://github.com/stricaud/faup.git", + "ref": "de31b6965fc4149c2095c7b721f456e428404736", + "subdirectory": "src/lib/bindings/python" + }, + "pymisp": { + "editable": true, + "git": "https://github.com/MISP/PyMISP.git", + "ref": "d4934cdf5f537c9f42ae37be7878de1848961de0" + }, + "python-dateutil": { + "hashes": [ + "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93", + "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02" + ], + "version": "==2.7.5" + }, + "python-magic": { + "hashes": [ + "sha256:f2674dcfad52ae6c49d4803fa027809540b130db1dec928cfbb9240316831375", + "sha256:f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5" + ], + "index": "pypi", + "version": "==0.4.15" + }, + "requests": { + "hashes": [ + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + ], + "version": "==2.21.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "urllib3": { + "hashes": [ + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + ], + "version": "==1.24.1" + } + }, + "develop": { + "certifi": { + "hashes": [ + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + ], + "version": "==2018.11.29" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "codecov": { + "hashes": [ + "sha256:8ed8b7c6791010d359baed66f84f061bba5bd41174bf324c31311e8737602788", + "sha256:ae00d68e18d8a20e9c3288ba3875ae03db3a8e892115bf9b83ef20507732bed4" + ], + "index": "pypi", + "version": "==2.0.15" + }, + "coverage": { + "hashes": [ + "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", + "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", + "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", + "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", + "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", + "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", + "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", + "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", + "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", + "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", + "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", + "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", + "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", + "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", + "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", + "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", + "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", + "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", + "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", + "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", + "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", + "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", + "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", + "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", + "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", + "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", + "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", + "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", + "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", + "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", + "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9" + ], + "index": "pypi", + "version": "==4.5.2" + }, + "coveralls": { + "hashes": [ + "sha256:ab638e88d38916a6cedbf80a9cd8992d5fa55c77ab755e262e00b36792b7cd6d", + "sha256:b2388747e2529fa4c669fb1e3e2756e4e07b6ee56c7d9fce05f35ccccc913aa0" + ], + "index": "pypi", + "version": "==1.5.1" + }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "nose": { + "hashes": [ + "sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac", + "sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a", + "sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98" + ], + "index": "pypi", + "version": "==1.3.7" + }, + "requests": { + "hashes": [ + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + ], + "version": "==2.21.0" + }, + "urllib3": { + "hashes": [ + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + ], + "version": "==1.24.1" + } + } +} diff --git a/requirements.txt b/requirements.txt index f08e8d0..56d37c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,16 @@ -dnspython -lief -python-magic -git+https://github.com/MISP/PyMISP.git -git+https://github.com/kbandla/pydeep.git -git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python -git+https://github.com/Rafiot/defang.git -aiosmtpd +-i https://pypi.org/simple +-e git+https://github.com/MISP/PyMISP.git@d4934cdf5f537c9f42ae37be7878de1848961de0#egg=pymisp +certifi==2018.11.29 +chardet==3.0.4 +dnspython==1.16.0 +git+https://github.com/Rafiot/defang.git@52972a25313e2899f98f1777b940cb2122566a26#egg=defang +git+https://github.com/kbandla/pydeep.git@bc0d33bff4b45718b4c5f2c79d4715d92a427eda#egg=pydeep +git+https://github.com/stricaud/faup.git@de31b6965fc4149c2095c7b721f456e428404736#egg=pyfaup&subdirectory=src/lib/bindings/python +idna==2.8 +jsonschema==2.6.0 +lief==0.9.0 +python-dateutil==2.7.5 +python-magic==0.4.15 +requests==2.21.0 +six==1.12.0 +urllib3==1.24.1 From 58ea8a64ca1f7c59c5211228988da4679de1d52d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 21 Jan 2019 14:39:04 +0100 Subject: [PATCH 44/50] fix: Support new version of pyfaup --- mail2misp/mail2misp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py index aa1709f..28210bd 100644 --- a/mail2misp/mail2misp.py +++ b/mail2misp/mail2misp.py @@ -228,20 +228,20 @@ class Mail2MISP(): ids_flag = True self.f.decode(entry) - domainname = self.f.get_domain().decode() + domainname = self.f.get_domain() if domainname in self.config.excludelist: # Ignore the entry continue - hostname = self.f.get_host().decode() + hostname = self.f.get_host() scheme = self.f.get_scheme() if scheme: - scheme = scheme.decode() + scheme = scheme resource_path = self.f.get_resource_path() if resource_path: - resource_path = resource_path.decode() + resource_path = resource_path if self.debug: syslog.syslog(domainname) @@ -298,7 +298,7 @@ class Mail2MISP(): comment = '' port = self.f.get_port() if port: - port = port.decode() + port = port comment = f'on port: {port}' if is_ip(hostname): From 84142f940226d4ba822f6375716a953ecd7eb2f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 21 Jan 2019 14:51:00 +0100 Subject: [PATCH 45/50] chg: Remove testing for python 3.7, lief from pypi doesn't support it. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 221e772..7f24b3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,6 @@ addons: python: - "3.6" - "3.6-dev" - - "3.7-dev" install: - git clone git://github.com/stricaud/faup.git From a4aea56cda756aac4e1990ff366b3ae90a49ae0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 21 Jan 2019 14:52:10 +0100 Subject: [PATCH 46/50] chg: Update readme to use pipenv --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b4a7dfb..730ff52 100644 --- a/README.md +++ b/README.md @@ -166,8 +166,10 @@ sudo make install # Update Shared libs sudo ldconfig +(sudo) pip install (--user) pipenv + # Install other python requirements -pip3 install -r requirements.txt +pipenv install # Test if the script is working ./mail_to_misp.py -h From b64d6512999b0d474825bfb70ea5d6f654056047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 25 Mar 2019 16:04:30 +0100 Subject: [PATCH 47/50] chg: Bump dependencies --- Pipfile.lock | 111 ++++++++++++++++++++++++++++----------------------- 1 file changed, 62 insertions(+), 49 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index c1ee946..f2a4131 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -16,12 +16,19 @@ ] }, "default": { + "attrs": { + "hashes": [ + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + ], + "version": "==19.1.0" + }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -51,10 +58,10 @@ }, "jsonschema": { "hashes": [ - "sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08", - "sha256:6ff5f3180870836cae40f06fa10419f557208175f13ad7bc26caa77beb1f6e02" + "sha256:0c0a81564f181de3212efa2d17de1910f8732fa1b71c42266d983cd74304e20d", + "sha256:a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a" ], - "version": "==2.6.0" + "version": "==3.0.1" }, "lief": { "hashes": [ @@ -69,20 +76,26 @@ }, "pyfaup": { "git": "https://github.com/stricaud/faup.git", - "ref": "de31b6965fc4149c2095c7b721f456e428404736", + "ref": "88dbbe2378552c9753b4f1e938663484909a4940", "subdirectory": "src/lib/bindings/python" }, "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "d4934cdf5f537c9f42ae37be7878de1848961de0" + "ref": "c888af177f88af653ad395924a3b840ca22f0af4" + }, + "pyrsistent": { + "hashes": [ + "sha256:3ca82748918eb65e2d89f222b702277099aca77e34843c5eb9d52451173970e2" + ], + "version": "==0.14.11" }, "python-dateutil": { "hashes": [ - "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93", - "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02" + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], - "version": "==2.7.5" + "version": "==2.8.0" }, "python-magic": { "hashes": [ @@ -117,10 +130,10 @@ "develop": { "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -139,48 +152,48 @@ }, "coverage": { "hashes": [ - "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", - "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", - "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", - "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", - "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", - "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", - "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", - "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", - "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", - "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", - "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", - "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", - "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", - "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", - "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", - "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", - "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", - "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", - "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", - "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", - "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", - "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", - "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", - "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", - "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", - "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", - "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", - "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", - "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", - "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", - "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9" + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" ], "index": "pypi", - "version": "==4.5.2" + "version": "==4.5.3" }, "coveralls": { "hashes": [ - "sha256:ab638e88d38916a6cedbf80a9cd8992d5fa55c77ab755e262e00b36792b7cd6d", - "sha256:b2388747e2529fa4c669fb1e3e2756e4e07b6ee56c7d9fce05f35ccccc913aa0" + "sha256:baa26648430d5c2225ab12d7e2067f75597a4b967034bba7e3d5ab7501d207a1", + "sha256:ff9b7823b15070f26f654837bb02a201d006baaf2083e0514ffd3b34a3ffed81" ], "index": "pypi", - "version": "==1.5.1" + "version": "==1.7.0" }, "docopt": { "hashes": [ From 5a73264518f35462e00f5b369145903ab5651a67 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Fri, 17 May 2019 12:19:14 +0900 Subject: [PATCH 48/50] fix: [pip] updated urllib3 CVE-2019-11324 --- Pipfile.lock | 6 +++--- requirements.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index f2a4131..c960d3d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -121,10 +121,10 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:a53063d8b9210a7bdec15e7b272776b9d42b2fd6816401a0d43006ad2f9902db", + "sha256:d363e3607d8de0c220d31950a8f38b18d5ba7c0830facd71a1c6b1036b7ce06c" ], - "version": "==1.24.1" + "version": "==1.25.2" } }, "develop": { diff --git a/requirements.txt b/requirements.txt index 56d37c8..89074ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,4 @@ python-dateutil==2.7.5 python-magic==0.4.15 requests==2.21.0 six==1.12.0 -urllib3==1.24.1 +urllib3>=1.24.2 From 73cda7fbf6c8137d06fe134fdee71af0dd7128ea Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Fri, 17 May 2019 12:23:33 +0900 Subject: [PATCH 49/50] fix: [pip] urllib3 fix --- Pipfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index c960d3d..b225dd0 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -226,10 +226,10 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:a53063d8b9210a7bdec15e7b272776b9d42b2fd6816401a0d43006ad2f9902db", + "sha256:d363e3607d8de0c220d31950a8f38b18d5ba7c0830facd71a1c6b1036b7ce06c" ], - "version": "==1.24.1" + "version": "==1.25.2" } } } From f84c9eadee32f07c438a61f6aa0932e928e7a301 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Thu, 18 Jul 2019 11:02:42 +0200 Subject: [PATCH 50/50] added new functionality to update an event --- mail2misp/mail2misp.py | 12 ++++++++++++ mail_to_misp.py | 7 ++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py index 28210bd..9df2987 100644 --- a/mail2misp/mail2misp.py +++ b/mail2misp/mail2misp.py @@ -367,3 +367,15 @@ class Mail2MISP(): for value, source in self.sightings_to_add: self.sighting(value, source) return event + + def update_event(self, event_id=None): + '''Update event on the remote MISP instance.''' + + if self.offline: + return self.misp_event.to_json() + event = self.misp.update_event(self.misp_event, event_id=event_id) + if self.config.sighting: + for value, source in self.sightings_to_add: + self.sighting(value, source) + return event + diff --git a/mail_to_misp.py b/mail_to_misp.py index 8b70161..7784e16 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -15,6 +15,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='Push a Mail into a MISP instance') parser.add_argument("-r", "--read", help="Read from tempfile.") parser.add_argument("-t", "--trap", action='store_true', default=False, help="Import the Email as-is.") + parser.add_argument("-e", "--event", default=False, help="Add indicators to this MISP event.") parser.add_argument('infile', nargs='?', type=argparse.FileType('rb')) args = parser.parse_args() @@ -66,5 +67,9 @@ if __name__ == '__main__': mail2misp.process_body_iocs() - mail2misp.add_event() + if args.event: + misp_event = args.event + mail2misp.update_event(event_id=misp_event) + else: + mail2misp.add_event() syslog.syslog("Job finished.")