diff --git a/.dockerignore b/.dockerignore
index 8eb1e4df8a..5670b8c15b 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -11,5 +11,6 @@
 !build_rust.py
 
 rust/target
+synapse/*.so
 
 **/__pycache__
diff --git a/changelog.d/13811.misc b/changelog.d/13811.misc
new file mode 100644
index 0000000000..7d8fb06a0f
--- /dev/null
+++ b/changelog.d/13811.misc
@@ -0,0 +1 @@
+Fix Docker build when Rust .so has been build locally first.
diff --git a/docker/Dockerfile b/docker/Dockerfile
index a057bf397b..b20951d4cf 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -31,7 +31,9 @@ ARG PYTHON_VERSION=3.9
 ###
 ### Stage 0: generate requirements.txt
 ###
-FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
+# We hardcode the use of Debian bullseye here because this could change upstream
+# and other Dockerfiles used for testing are expecting bullseye.
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as requirements
 
 # RUN --mount is specific to buildkit and is documented at
 # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
@@ -76,7 +78,7 @@ RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
 ###
 ### Stage 1: builder
 ###
-FROM docker.io/python:${PYTHON_VERSION}-slim as builder
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as builder
 
 # install the OS build deps
 RUN \
@@ -137,7 +139,7 @@ RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
 ### Stage 2: runtime
 ###
 
-FROM docker.io/python:${PYTHON_VERSION}-slim
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye
 
 LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
 LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'