diff --git a/.travis.yml b/.travis.yml
index fd41841c77..655fab9d8e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,6 +23,9 @@ branches:
     - develop
     - /^release-v/
 
+# When running the tox environments that call Twisted Trial, we can pass the -j
+# flag to run the tests concurrently. We set this to 2 for CPU bound tests
+# (SQLite) and 4 for I/O bound tests (PostgreSQL).
 matrix:
   fast_finish: true
   include:
@@ -33,10 +36,10 @@ matrix:
     env: TOX_ENV="pep8,check_isort"
 
   - python: 2.7
-    env: TOX_ENV=py27
+    env: TOX_ENV=py27 TRIAL_FLAGS="-j 2"
 
   - python: 2.7
-    env: TOX_ENV=py27-old
+    env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2"
 
   - python: 2.7
     env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
@@ -44,10 +47,10 @@ matrix:
       - postgresql
 
   - python: 3.5
-    env: TOX_ENV=py35
+    env: TOX_ENV=py35 TRIAL_FLAGS="-j 2"
 
   - python: 3.6
-    env: TOX_ENV=py36
+    env: TOX_ENV=py36 TRIAL_FLAGS="-j 2"
 
   - python: 3.6
     env: TOX_ENV=py36-postgres TRIAL_FLAGS="-j 4"
diff --git a/changelog.d/4095.bugfix b/changelog.d/4095.bugfix
new file mode 100644
index 0000000000..76ee7148c2
--- /dev/null
+++ b/changelog.d/4095.bugfix
@@ -0,0 +1 @@
+Fix exceptions when using the email mailer on Python 3.
diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py
index f369124258..50e1007d84 100644
--- a/synapse/push/emailpusher.py
+++ b/synapse/push/emailpusher.py
@@ -85,7 +85,10 @@ class EmailPusher(object):
             self.timed_call = None
 
     def on_new_notifications(self, min_stream_ordering, max_stream_ordering):
-        self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering)
+        if self.max_stream_ordering:
+            self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering)
+        else:
+            self.max_stream_ordering = max_stream_ordering
         self._start_processing()
 
     def on_new_receipts(self, min_stream_id, max_stream_id):
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index 16fb5e8471..ebcb93bfc7 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -26,7 +26,6 @@ import bleach
 import jinja2
 
 from twisted.internet import defer
-from twisted.mail.smtp import sendmail
 
 from synapse.api.constants import EventTypes
 from synapse.api.errors import StoreError
@@ -85,6 +84,7 @@ class Mailer(object):
         self.notif_template_html = notif_template_html
         self.notif_template_text = notif_template_text
 
+        self.sendmail = self.hs.get_sendmail()
         self.store = self.hs.get_datastore()
         self.macaroon_gen = self.hs.get_macaroon_generator()
         self.state_handler = self.hs.get_state_handler()
@@ -191,11 +191,11 @@ class Mailer(object):
         multipart_msg.attach(html_part)
 
         logger.info("Sending email push notification to %s" % email_address)
-        # logger.debug(html_text)
 
-        yield sendmail(
+        yield self.sendmail(
             self.hs.config.email_smtp_host,
-            raw_from, raw_to, multipart_msg.as_string(),
+            raw_from, raw_to, multipart_msg.as_string().encode('utf8'),
+            reactor=self.hs.get_reactor(),
             port=self.hs.config.email_smtp_port,
             requireAuthentication=self.hs.config.email_smtp_user is not None,
             username=self.hs.config.email_smtp_user,
@@ -333,7 +333,7 @@ class Mailer(object):
                           notif_events, user_id, reason):
         if len(notifs_by_room) == 1:
             # Only one room has new stuff
-            room_id = notifs_by_room.keys()[0]
+            room_id = list(notifs_by_room.keys())[0]
 
             # If the room has some kind of name, use it, but we don't
             # want the generated-from-names one here otherwise we'll
diff --git a/synapse/server.py b/synapse/server.py
index cf6b872cbd..9985687b95 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -23,6 +23,7 @@ import abc
 import logging
 
 from twisted.enterprise import adbapi
+from twisted.mail.smtp import sendmail
 from twisted.web.client import BrowserLikePolicyForHTTPS
 
 from synapse.api.auth import Auth
@@ -174,6 +175,7 @@ class HomeServer(object):
         'message_handler',
         'pagination_handler',
         'room_context_handler',
+        'sendmail',
     ]
 
     # This is overridden in derived application classes
@@ -269,6 +271,9 @@ class HomeServer(object):
     def build_room_creation_handler(self):
         return RoomCreationHandler(self)
 
+    def build_sendmail(self):
+        return sendmail
+
     def build_state_handler(self):
         return StateHandler(self)
 
diff --git a/tests/push/__init__.py b/tests/push/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/push/test_email.py b/tests/push/test_email.py
new file mode 100644
index 0000000000..50ee6910d1
--- /dev/null
+++ b/tests/push/test_email.py
@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import pkg_resources
+
+from twisted.internet.defer import Deferred
+
+from synapse.rest.client.v1 import admin, login, room
+
+from tests.unittest import HomeserverTestCase
+
+try:
+    from synapse.push.mailer import load_jinja2_templates
+except Exception:
+    load_jinja2_templates = None
+
+
+class EmailPusherTests(HomeserverTestCase):
+
+    skip = "No Jinja installed" if not load_jinja2_templates else None
+    servlets = [
+        admin.register_servlets,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+    user_id = True
+    hijack_auth = False
+
+    def make_homeserver(self, reactor, clock):
+
+        # List[Tuple[Deferred, args, kwargs]]
+        self.email_attempts = []
+
+        def sendmail(*args, **kwargs):
+            d = Deferred()
+            self.email_attempts.append((d, args, kwargs))
+            return d
+
+        config = self.default_config()
+        config.email_enable_notifs = True
+        config.start_pushers = True
+
+        config.email_template_dir = os.path.abspath(
+            pkg_resources.resource_filename('synapse', 'res/templates')
+        )
+        config.email_notif_template_html = "notif_mail.html"
+        config.email_notif_template_text = "notif_mail.txt"
+        config.email_smtp_host = "127.0.0.1"
+        config.email_smtp_port = 20
+        config.require_transport_security = False
+        config.email_smtp_user = None
+        config.email_app_name = "Matrix"
+        config.email_notif_from = "test@example.com"
+
+        hs = self.setup_test_homeserver(config=config, sendmail=sendmail)
+
+        return hs
+
+    def test_sends_email(self):
+
+        # Register the user who gets notified
+        user_id = self.register_user("user", "pass")
+        access_token = self.login("user", "pass")
+
+        # Register the user who sends the message
+        other_user_id = self.register_user("otheruser", "pass")
+        other_access_token = self.login("otheruser", "pass")
+
+        # Register the pusher
+        user_tuple = self.get_success(
+            self.hs.get_datastore().get_user_by_access_token(access_token)
+        )
+        token_id = user_tuple["token_id"]
+
+        self.get_success(
+            self.hs.get_pusherpool().add_pusher(
+                user_id=user_id,
+                access_token=token_id,
+                kind="email",
+                app_id="m.email",
+                app_display_name="Email Notifications",
+                device_display_name="a@example.com",
+                pushkey="a@example.com",
+                lang=None,
+                data={},
+            )
+        )
+
+        # Create a room
+        room = self.helper.create_room_as(user_id, tok=access_token)
+
+        # Invite the other person
+        self.helper.invite(room=room, src=user_id, tok=access_token, targ=other_user_id)
+
+        # The other user joins
+        self.helper.join(room=room, user=other_user_id, tok=other_access_token)
+
+        # The other user sends some messages
+        self.helper.send(room, body="Hi!", tok=other_access_token)
+        self.helper.send(room, body="There!", tok=other_access_token)
+
+        # Get the stream ordering before it gets sent
+        pushers = self.get_success(
+            self.hs.get_datastore().get_pushers_by(dict(user_name=user_id))
+        )
+        self.assertEqual(len(pushers), 1)
+        last_stream_ordering = pushers[0]["last_stream_ordering"]
+
+        # Advance time a bit, so the pusher will register something has happened
+        self.pump(100)
+
+        # It hasn't succeeded yet, so the stream ordering shouldn't have moved
+        pushers = self.get_success(
+            self.hs.get_datastore().get_pushers_by(dict(user_name=user_id))
+        )
+        self.assertEqual(len(pushers), 1)
+        self.assertEqual(last_stream_ordering, pushers[0]["last_stream_ordering"])
+
+        # One email was attempted to be sent
+        self.assertEqual(len(self.email_attempts), 1)
+
+        # Make the email succeed
+        self.email_attempts[0][0].callback(True)
+        self.pump()
+
+        # One email was attempted to be sent
+        self.assertEqual(len(self.email_attempts), 1)
+
+        # The stream ordering has increased
+        pushers = self.get_success(
+            self.hs.get_datastore().get_pushers_by(dict(user_name=user_id))
+        )
+        self.assertEqual(len(pushers), 1)
+        self.assertTrue(pushers[0]["last_stream_ordering"] > last_stream_ordering)
diff --git a/tests/server.py b/tests/server.py
index 7bee58dff1..819c854448 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -125,7 +125,9 @@ def make_request(method, path, content=b"", access_token=None, request=SynapseRe
     req.content = BytesIO(content)
 
     if access_token:
-        req.requestHeaders.addRawHeader(b"Authorization", b"Bearer " + access_token)
+        req.requestHeaders.addRawHeader(
+            b"Authorization", b"Bearer " + access_token.encode('ascii')
+        )
 
     if content:
         req.requestHeaders.addRawHeader(b"Content-Type", b"application/json")
diff --git a/tests/test_mau.py b/tests/test_mau.py
index bdbacb8448..5d387851c5 100644
--- a/tests/test_mau.py
+++ b/tests/test_mau.py
@@ -207,7 +207,7 @@ class TestMauLimit(unittest.TestCase):
 
     def do_sync_for_user(self, token):
         request, channel = make_request(
-            "GET", "/sync", access_token=token.encode('ascii')
+            "GET", "/sync", access_token=token
         )
         render(request, self.resource, self.reactor)
 
diff --git a/tests/unittest.py b/tests/unittest.py
index a59291cc60..4d40bdb6a5 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -146,6 +146,13 @@ def DEBUG(target):
     return target
 
 
+def INFO(target):
+    """A decorator to set the .loglevel attribute to logging.INFO.
+    Can apply to either a TestCase or an individual test method."""
+    target.loglevel = logging.INFO
+    return target
+
+
 class HomeserverTestCase(TestCase):
     """
     A base TestCase that reduces boilerplate for HomeServer-using test cases.
@@ -373,5 +380,5 @@ class HomeserverTestCase(TestCase):
         self.render(request)
         self.assertEqual(channel.code, 200)
 
-        access_token = channel.json_body["access_token"].encode('ascii')
+        access_token = channel.json_body["access_token"]
         return access_token