diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 369a1ffed1..44b258dca6 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -90,6 +90,17 @@ steps:
           image: "python:3.7"
           propagate-environment: true
 
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py27-old,codecov"
+    label: ":python: 2.7 / SQLite / Old Deps"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:2.7"
+          propagate-environment: true
+
   - label: ":python: 2.7 / :postgres: 9.4"
     env:
       TRIAL_FLAGS: "-j 4"
diff --git a/changelog.d/4879.misc b/changelog.d/4879.misc
new file mode 100644
index 0000000000..574017230c
--- /dev/null
+++ b/changelog.d/4879.misc
@@ -0,0 +1 @@
+Reinstate test case that runs unit tests against oldest supported dependencies.
diff --git a/tox.ini b/tox.ini
index 19080a648f..ef543890f9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -82,15 +82,23 @@ deps =
 
     mock
     lxml
+    coverage
+
 commands =
     /usr/bin/find "{toxinidir}" -name '*.pyc' -delete
     # Make all greater-thans equals so we test the oldest version of our direct
     # dependencies, but make the pyopenssl 17.0, which can work against an
     # OpenSSL 1.1 compiled cryptography (as older ones don't compile on Travis).
     /bin/sh -c 'python -m synapse.python_dependencies | sed -e "s/>=/==/g" -e "s/psycopg2==2.6//" -e "s/pyopenssl==16.0.0/pyopenssl==17.0.0/" | xargs pip install'
+
+    # Add this so that coverage will run on subprocesses
+    /bin/sh -c 'echo "import coverage; coverage.process_startup()" > {envsitepackagesdir}/../sitecustomize.py'
+
     # Install Synapse itself. This won't update any libraries.
     pip install -e .
-    {envbindir}/trial {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
+
+    {envbindir}/coverage run "{envbindir}/trial"  {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
+
 
 [testenv:packaging]
 skip_install=True